ftl/FTLLowerDFGToLLVM.cpp

   1 /*
   2  * Copyright (C) 2013-2015 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "FTLLowerDFGToLLVM.h"
  28
  29 #if ENABLE(FTL_JIT)
  30
  31 #include "CodeBlockWithJITType.h"
  32 #include "DFGAbstractInterpreterInlines.h"
  33 #include "DFGInPlaceAbstractState.h"
  34 #include "DFGOSRAvailabilityAnalysisPhase.h"
  35 #include "DFGOSRExitFuzz.h"
  36 #include "DirectArguments.h"
  37 #include "FTLAbstractHeapRepository.h"
  38 #include "FTLAvailableRecovery.h"
  39 #include "FTLForOSREntryJITCode.h"
  40 #include "FTLFormattedValue.h"
  41 #include "FTLInlineCacheSize.h"
  42 #include "FTLLoweredNodeValue.h"
  43 #include "FTLOperations.h"
  44 #include "FTLOutput.h"
  45 #include "FTLThunks.h"
  46 #include "FTLWeightedTarget.h"
  47 #include "JSCInlines.h"
  48 #include "JSLexicalEnvironment.h"
  49 #include "OperandsInlines.h"
  50 #include "ScopedArguments.h"
  51 #include "ScopedArgumentsTable.h"
  52 #include "VirtualRegister.h"
  53 #include <atomic>
  54 #include <dlfcn.h>
  55 #include <llvm/InitializeLLVM.h>
  56 #include <unordered_set>
  57 #include <wtf/ProcessID.h>
  58
  59 #if ENABLE(FTL_NATIVE_CALL_INLINING)
  60 #include "BundlePath.h"
  61 #endif
  62
  63 namespace JSC { namespace FTL {
  64
  65 using namespace DFG;
  66
  67 namespace {
  68
  69 std::atomic<int> compileCounter;
  70
  71 #if ASSERT_DISABLED
  72 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable()
  73 {
  74     CRASH();
  75 }
  76 #else
  77 NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
  78     CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
  79 {
  80     dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
  81     if (nodeIndex != UINT_MAX)
  82         dataLog(", node @", nodeIndex);
  83     dataLog(".\n");
  84     CRASH();
  85 }
  86 #endif
  87
  88 // Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
  89 // significantly less dead code.
  90 #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
  91         FormattedValue _ftc_lowValue = (lowValue);                      \
  92         Edge _ftc_highValue = (highValue);                              \
  93         SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough);  \
  94         if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
  95             break;                                                      \
  96         typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
  97     } while (false)
  98
  99 class LowerDFGToLLVM {
 100 public:
 101     LowerDFGToLLVM(State& state)
 102         : m_graph(state.graph)
 103         , m_ftlState(state)
 104         , m_heaps(state.context)
 105         , m_out(state.context)
 106         , m_state(state.graph)
 107         , m_interpreter(state.graph, m_state)
 108         , m_stackmapIDs(0)
 109         , m_tbaaKind(mdKindID(state.context, "tbaa"))
 110         , m_tbaaStructKind(mdKindID(state.context, "tbaa.struct"))
 111     {
 112     }
 113
 114     void lower()
 115     {
 116         CString name;
 117         if (verboseCompilationEnabled()) {
 118             name = toCString(
 119                 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
 120                 "_", codeBlock()->hash());
 121         } else
 122             name = "jsBody";
 123
 124         m_graph.m_dominators.computeIfNecessary(m_graph);
 125
 126         m_ftlState.module =
 127             moduleCreateWithNameInContext(name.data(), m_ftlState.context);
 128
 129         m_ftlState.function = addFunction(
 130             m_ftlState.module, name.data(), functionType(m_out.int64));
 131         setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
 132         if (isX86() && Options::llvmDisallowAVX()) {
 133             // AVX makes V8/raytrace 80% slower. It makes Kraken/audio-oscillator 4.5x
 134             // slower. It should be disabled.
 135             addTargetDependentFunctionAttr(m_ftlState.function, "target-features", "-avx");
 136         }
 137
 138         if (verboseCompilationEnabled())
 139             dataLog("Function ready, beginning lowering.\n");
 140
 141         m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
 142
 143         m_prologue = FTL_NEW_BLOCK(m_out, ("Prologue"));
 144         LBasicBlock stackOverflow = FTL_NEW_BLOCK(m_out, ("Stack overflow"));
 145         m_handleExceptions = FTL_NEW_BLOCK(m_out, ("Handle Exceptions"));
 146
 147         LBasicBlock checkArguments = FTL_NEW_BLOCK(m_out, ("Check arguments"));
 148
 149         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
 150             m_highBlock = m_graph.block(blockIndex);
 151             if (!m_highBlock)
 152                 continue;
 153             m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
 154         }
 155
 156         m_out.appendTo(m_prologue, stackOverflow);
 157         createPhiVariables();
 158
 159         auto preOrder = m_graph.blocksInPreOrder();
 160
 161         int maxNumberOfArguments = -1;
 162         for (BasicBlock* block : preOrder) {
 163             for (unsigned nodeIndex = block->size(); nodeIndex--; ) {
 164                 Node* node = block->at(nodeIndex);
 165                 switch (node->op()) {
 166                 case NativeCall:
 167                 case NativeConstruct: {
 168                     int numArgs = node->numChildren();
 169                     if (numArgs > maxNumberOfArguments)
 170                         maxNumberOfArguments = numArgs;
 171                     break;
 172                 }
 173                 default:
 174                     break;
 175                 }
 176             }
 177         }
 178
 179         if (maxNumberOfArguments >= 0) {
 180             m_execState = m_out.alloca(arrayType(m_out.int64, JSStack::CallFrameHeaderSize + maxNumberOfArguments));
 181             m_execStorage = m_out.ptrToInt(m_execState, m_out.intPtr);
 182         }
 183
 184         LValue capturedAlloca = m_out.alloca(arrayType(m_out.int64, m_graph.m_nextMachineLocal));
 185
 186         m_captured = m_out.add(
 187             m_out.ptrToInt(capturedAlloca, m_out.intPtr),
 188             m_out.constIntPtr(m_graph.m_nextMachineLocal * sizeof(Register)));
 189
 190         m_ftlState.capturedStackmapID = m_stackmapIDs++;
 191         m_out.call(
 192             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.capturedStackmapID),
 193             m_out.int32Zero, capturedAlloca);
 194
 195         // If we have any CallVarargs then we nee to have a spill slot for it.
 196         bool hasVarargs = false;
 197         for (BasicBlock* block : preOrder) {
 198             for (Node* node : *block) {
 199                 switch (node->op()) {
 200                 case CallVarargs:
 201                 case CallForwardVarargs:
 202                 case ConstructVarargs:
 203                 case ConstructForwardVarargs:
 204                     hasVarargs = true;
 205                     break;
 206                 default:
 207                     break;
 208                 }
 209             }
 210         }
 211         if (hasVarargs) {
 212             LValue varargsSpillSlots = m_out.alloca(
 213                 arrayType(m_out.int64, JSCallVarargs::numSpillSlotsNeeded()));
 214             m_ftlState.varargsSpillSlotsStackmapID = m_stackmapIDs++;
 215             m_out.call(
 216                 m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.varargsSpillSlotsStackmapID),
 217                 m_out.int32Zero, varargsSpillSlots);
 218         }
 219
 220         // We should not create any alloca's after this point, since they will cease to
 221         // be mem2reg candidates.
 222
 223         m_callFrame = m_out.ptrToInt(
 224             m_out.call(m_out.frameAddressIntrinsic(), m_out.int32Zero), m_out.intPtr);
 225         m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
 226         m_tagMask = m_out.constInt64(TagMask);
 227
 228         m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));
 229
 230         m_out.branch(
 231             didOverflowStack(), rarely(stackOverflow), usually(checkArguments));
 232
 233         m_out.appendTo(stackOverflow, m_handleExceptions);
 234         m_out.call(m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));
 235         m_ftlState.handleStackOverflowExceptionStackmapID = m_stackmapIDs++;
 236         m_out.call(
 237             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleStackOverflowExceptionStackmapID),
 238             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 239         m_out.unreachable();
 240
 241         m_out.appendTo(m_handleExceptions, checkArguments);
 242         m_ftlState.handleExceptionStackmapID = m_stackmapIDs++;
 243         m_out.call(
 244             m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleExceptionStackmapID),
 245             m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
 246         m_out.unreachable();
 247
 248         m_out.appendTo(checkArguments, lowBlock(m_graph.block(0)));
 249         availabilityMap().clear();
 250         availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
 251         for (unsigned i = codeBlock()->numParameters(); i--;) {
 252             availabilityMap().m_locals.argument(i) =
 253                 Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
 254         }
 255         m_codeOriginForExitTarget = CodeOrigin(0);
 256         m_codeOriginForExitProfile = CodeOrigin(0);
 257         m_node = nullptr;
 258         for (unsigned i = codeBlock()->numParameters(); i--;) {
 259             Node* node = m_graph.m_arguments[i];
 260             VirtualRegister operand = virtualRegisterForArgument(i);
 261
 262             LValue jsValue = m_out.load64(addressFor(operand));
 263
 264             if (node) {
 265                 DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal);
 266
 267                 // This is a hack, but it's an effective one. It allows us to do CSE on the
 268                 // primordial load of arguments. This assumes that the GetLocal that got put in
 269                 // place of the original SetArgument doesn't have any effects before it. This
 270                 // should hold true.
 271                 m_loadedArgumentValues.add(node, jsValue);
 272             }
 273
 274             switch (m_graph.m_argumentFormats[i]) {
 275             case FlushedInt32:
 276                 speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue));
 277                 break;
 278             case FlushedBoolean:
 279                 speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue));
 280                 break;
 281             case FlushedCell:
 282                 speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue));
 283                 break;
 284             case FlushedJSValue:
 285                 break;
 286             default:
 287                 DFG_CRASH(m_graph, node, "Bad flush format for argument");
 288                 break;
 289             }
 290         }
 291         m_out.jump(lowBlock(m_graph.block(0)));
 292
 293         for (BasicBlock* block : preOrder)
 294             compileBlock(block);
 295
 296         if (Options::dumpLLVMIR())
 297             dumpModule(m_ftlState.module);
 298
 299         if (verboseCompilationEnabled())
 300             m_ftlState.dumpState("after lowering");
 301         if (validationEnabled())
 302             verifyModule(m_ftlState.module);
 303     }
 304
 305 private:
 306
 307     void createPhiVariables()
 308     {
 309         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 310             BasicBlock* block = m_graph.block(blockIndex);
 311             if (!block)
 312                 continue;
 313             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
 314                 Node* node = block->at(nodeIndex);
 315                 if (node->op() != Phi)
 316                     continue;
 317                 LType type;
 318                 switch (node->flags() & NodeResultMask) {
 319                 case NodeResultDouble:
 320                     type = m_out.doubleType;
 321                     break;
 322                 case NodeResultInt32:
 323                     type = m_out.int32;
 324                     break;
 325                 case NodeResultInt52:
 326                     type = m_out.int64;
 327                     break;
 328                 case NodeResultBoolean:
 329                     type = m_out.boolean;
 330                     break;
 331                 case NodeResultJS:
 332                     type = m_out.int64;
 333                     break;
 334                 default:
 335                     DFG_CRASH(m_graph, node, "Bad Phi node result type");
 336                     break;
 337                 }
 338                 m_phis.add(node, buildAlloca(m_out.m_builder, type));
 339             }
 340         }
 341     }
 342
 343     void compileBlock(BasicBlock* block)
 344     {
 345         if (!block)
 346             return;
 347
 348         if (verboseCompilationEnabled())
 349             dataLog("Compiling block ", *block, "\n");
 350
 351         m_highBlock = block;
 352
 353         LBasicBlock lowBlock = m_blocks.get(m_highBlock);
 354
 355         m_nextHighBlock = 0;
 356         for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
 357             m_nextHighBlock = m_graph.block(nextBlockIndex);
 358             if (m_nextHighBlock)
 359                 break;
 360         }
 361         m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
 362
 363         // All of this effort to find the next block gives us the ability to keep the
 364         // generated IR in roughly program order. This ought not affect the performance
 365         // of the generated code (since we expect LLVM to reorder things) but it will
 366         // make IR dumps easier to read.
 367         m_out.appendTo(lowBlock, m_nextLowBlock);
 368
 369         if (Options::ftlCrashes())
 370             m_out.trap();
 371
 372         if (!m_highBlock->cfaHasVisited) {
 373             if (verboseCompilationEnabled())
 374                 dataLog("Bailing because CFA didn't reach.\n");
 375             crash(m_highBlock->index, UINT_MAX);
 376             return;
 377         }
 378
 379         m_availabilityCalculator.beginBlock(m_highBlock);
 380
 381         m_state.reset();
 382         m_state.beginBasicBlock(m_highBlock);
 383
 384         for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
 385             if (!compileNode(m_nodeIndex))
 386                 break;
 387         }
 388     }
 389
 390     void safelyInvalidateAfterTermination()
 391     {
 392         if (verboseCompilationEnabled())
 393             dataLog("Bailing.\n");
 394         crash();
 395
 396         // Invalidate dominated blocks. Under normal circumstances we would expect
 397         // them to be invalidated already. But you can have the CFA become more
 398         // precise over time because the structures of objects change on the main
 399         // thread. Failing to do this would result in weird crashes due to a value
 400         // being used but not defined. Race conditions FTW!
 401         for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
 402             BasicBlock* target = m_graph.block(blockIndex);
 403             if (!target)
 404                 continue;
 405             if (m_graph.m_dominators.dominates(m_highBlock, target)) {
 406                 if (verboseCompilationEnabled())
 407                     dataLog("Block ", *target, " will bail also.\n");
 408                 target->cfaHasVisited = false;
 409             }
 410         }
 411     }
 412
 413     bool compileNode(unsigned nodeIndex)
 414     {
 415         if (!m_state.isValid()) {
 416             safelyInvalidateAfterTermination();
 417             return false;
 418         }
 419
 420         m_node = m_highBlock->at(nodeIndex);
 421         m_codeOriginForExitProfile = m_node->origin.semantic;
 422         m_codeOriginForExitTarget = m_node->origin.forExit;
 423
 424         if (verboseCompilationEnabled())
 425             dataLog("Lowering ", m_node, "\n");
 426
 427         m_availableRecoveries.resize(0);
 428
 429         m_interpreter.startExecuting();
 430
 431         switch (m_node->op()) {
 432         case Upsilon:
 433             compileUpsilon();
 434             break;
 435         case Phi:
 436             compilePhi();
 437             break;
 438         case JSConstant:
 439             break;
 440         case DoubleConstant:
 441             compileDoubleConstant();
 442             break;
 443         case Int52Constant:
 444             compileInt52Constant();
 445             break;
 446         case DoubleRep:
 447             compileDoubleRep();
 448             break;
 449         case DoubleAsInt32:
 450             compileDoubleAsInt32();
 451             break;
 452         case ValueRep:
 453             compileValueRep();
 454             break;
 455         case Int52Rep:
 456             compileInt52Rep();
 457             break;
 458         case ValueToInt32:
 459             compileValueToInt32();
 460             break;
 461         case BooleanToNumber:
 462             compileBooleanToNumber();
 463             break;
 464         case ExtractOSREntryLocal:
 465             compileExtractOSREntryLocal();
 466             break;
 467         case GetStack:
 468             compileGetStack();
 469             break;
 470         case PutStack:
 471             compilePutStack();
 472             break;
 473         case Check:
 474             compileNoOp();
 475             break;
 476         case ToThis:
 477             compileToThis();
 478             break;
 479         case ValueAdd:
 480             compileValueAdd();
 481             break;
 482         case ArithAdd:
 483         case ArithSub:
 484             compileArithAddOrSub();
 485             break;
 486         case ArithClz32:
 487             compileArithClz32();
 488             break;
 489         case ArithMul:
 490             compileArithMul();
 491             break;
 492         case ArithDiv:
 493             compileArithDiv();
 494             break;
 495         case ArithMod:
 496             compileArithMod();
 497             break;
 498         case ArithMin:
 499         case ArithMax:
 500             compileArithMinOrMax();
 501             break;
 502         case ArithAbs:
 503             compileArithAbs();
 504             break;
 505         case ArithSin:
 506             compileArithSin();
 507             break;
 508         case ArithCos:
 509             compileArithCos();
 510             break;
 511         case ArithPow:
 512             compileArithPow();
 513             break;
 514         case ArithRound:
 515             compileArithRound();
 516             break;
 517         case ArithSqrt:
 518             compileArithSqrt();
 519             break;
 520         case ArithLog:
 521             compileArithLog();
 522             break;
 523         case ArithFRound:
 524             compileArithFRound();
 525             break;
 526         case ArithNegate:
 527             compileArithNegate();
 528             break;
 529         case BitAnd:
 530             compileBitAnd();
 531             break;
 532         case BitOr:
 533             compileBitOr();
 534             break;
 535         case BitXor:
 536             compileBitXor();
 537             break;
 538         case BitRShift:
 539             compileBitRShift();
 540             break;
 541         case BitLShift:
 542             compileBitLShift();
 543             break;
 544         case BitURShift:
 545             compileBitURShift();
 546             break;
 547         case UInt32ToNumber:
 548             compileUInt32ToNumber();
 549             break;
 550         case CheckStructure:
 551             compileCheckStructure();
 552             break;
 553         case CheckCell:
 554             compileCheckCell();
 555             break;
 556         case CheckNotEmpty:
 557             compileCheckNotEmpty();
 558             break;
 559         case CheckBadCell:
 560             compileCheckBadCell();
 561             break;
 562         case GetExecutable:
 563             compileGetExecutable();
 564             break;
 565         case ArrayifyToStructure:
 566             compileArrayifyToStructure();
 567             break;
 568         case PutStructure:
 569             compilePutStructure();
 570             break;
 571         case GetById:
 572             compileGetById();
 573             break;
 574         case In:
 575             compileIn();
 576             break;
 577         case PutById:
 578         case PutByIdDirect:
 579             compilePutById();
 580             break;
 581         case GetButterfly:
 582             compileGetButterfly();
 583             break;
 584         case ConstantStoragePointer:
 585             compileConstantStoragePointer();
 586             break;
 587         case GetIndexedPropertyStorage:
 588             compileGetIndexedPropertyStorage();
 589             break;
 590         case CheckArray:
 591             compileCheckArray();
 592             break;
 593         case GetArrayLength:
 594             compileGetArrayLength();
 595             break;
 596         case CheckInBounds:
 597             compileCheckInBounds();
 598             break;
 599         case GetByVal:
 600             compileGetByVal();
 601             break;
 602         case GetMyArgumentByVal:
 603             compileGetMyArgumentByVal();
 604             break;
 605         case PutByVal:
 606         case PutByValAlias:
 607         case PutByValDirect:
 608             compilePutByVal();
 609             break;
 610         case ArrayPush:
 611             compileArrayPush();
 612             break;
 613         case ArrayPop:
 614             compileArrayPop();
 615             break;
 616         case CreateActivation:
 617             compileCreateActivation();
 618             break;
 619         case NewFunction:
 620             compileNewFunction();
 621             break;
 622         case CreateDirectArguments:
 623             compileCreateDirectArguments();
 624             break;
 625         case CreateScopedArguments:
 626             compileCreateScopedArguments();
 627             break;
 628         case CreateClonedArguments:
 629             compileCreateClonedArguments();
 630             break;
 631         case NewObject:
 632             compileNewObject();
 633             break;
 634         case NewArray:
 635             compileNewArray();
 636             break;
 637         case NewArrayBuffer:
 638             compileNewArrayBuffer();
 639             break;
 640         case NewArrayWithSize:
 641             compileNewArrayWithSize();
 642             break;
 643         case GetTypedArrayByteOffset:
 644             compileGetTypedArrayByteOffset();
 645             break;
 646         case AllocatePropertyStorage:
 647             compileAllocatePropertyStorage();
 648             break;
 649         case ReallocatePropertyStorage:
 650             compileReallocatePropertyStorage();
 651             break;
 652         case ToString:
 653         case CallStringConstructor:
 654             compileToStringOrCallStringConstructor();
 655             break;
 656         case ToPrimitive:
 657             compileToPrimitive();
 658             break;
 659         case MakeRope:
 660             compileMakeRope();
 661             break;
 662         case StringCharAt:
 663             compileStringCharAt();
 664             break;
 665         case StringCharCodeAt:
 666             compileStringCharCodeAt();
 667             break;
 668         case GetByOffset:
 669         case GetGetterSetterByOffset:
 670             compileGetByOffset();
 671             break;
 672         case GetGetter:
 673             compileGetGetter();
 674             break;
 675         case GetSetter:
 676             compileGetSetter();
 677             break;
 678         case MultiGetByOffset:
 679             compileMultiGetByOffset();
 680             break;
 681         case PutByOffset:
 682             compilePutByOffset();
 683             break;
 684         case MultiPutByOffset:
 685             compileMultiPutByOffset();
 686             break;
 687         case GetGlobalVar:
 688             compileGetGlobalVar();
 689             break;
 690         case PutGlobalVar:
 691             compilePutGlobalVar();
 692             break;
 693         case NotifyWrite:
 694             compileNotifyWrite();
 695             break;
 696         case GetCallee:
 697             compileGetCallee();
 698             break;
 699         case GetArgumentCount:
 700             compileGetArgumentCount();
 701             break;
 702         case GetScope:
 703             compileGetScope();
 704             break;
 705         case SkipScope:
 706             compileSkipScope();
 707             break;
 708         case GetClosureVar:
 709             compileGetClosureVar();
 710             break;
 711         case PutClosureVar:
 712             compilePutClosureVar();
 713             break;
 714         case GetFromArguments:
 715             compileGetFromArguments();
 716             break;
 717         case PutToArguments:
 718             compilePutToArguments();
 719             break;
 720         case CompareEq:
 721             compileCompareEq();
 722             break;
 723         case CompareEqConstant:
 724             compileCompareEqConstant();
 725             break;
 726         case CompareStrictEq:
 727             compileCompareStrictEq();
 728             break;
 729         case CompareLess:
 730             compileCompareLess();
 731             break;
 732         case CompareLessEq:
 733             compileCompareLessEq();
 734             break;
 735         case CompareGreater:
 736             compileCompareGreater();
 737             break;
 738         case CompareGreaterEq:
 739             compileCompareGreaterEq();
 740             break;
 741         case LogicalNot:
 742             compileLogicalNot();
 743             break;
 744         case Call:
 745         case Construct:
 746             compileCallOrConstruct();
 747             break;
 748         case CallVarargs:
 749         case CallForwardVarargs:
 750         case ConstructVarargs:
 751         case ConstructForwardVarargs:
 752             compileCallOrConstructVarargs();
 753             break;
 754         case LoadVarargs:
 755             compileLoadVarargs();
 756             break;
 757         case ForwardVarargs:
 758             compileForwardVarargs();
 759             break;
 760 #if ENABLE(FTL_NATIVE_CALL_INLINING)
 761         case NativeCall:
 762         case NativeConstruct:
 763             compileNativeCallOrConstruct();
 764             break;
 765 #endif
 766         case Jump:
 767             compileJump();
 768             break;
 769         case Branch:
 770             compileBranch();
 771             break;
 772         case Switch:
 773             compileSwitch();
 774             break;
 775         case Return:
 776             compileReturn();
 777             break;
 778         case ForceOSRExit:
 779             compileForceOSRExit();
 780             break;
 781         case Throw:
 782         case ThrowReferenceError:
 783             compileThrow();
 784             break;
 785         case InvalidationPoint:
 786             compileInvalidationPoint();
 787             break;
 788         case IsUndefined:
 789             compileIsUndefined();
 790             break;
 791         case IsBoolean:
 792             compileIsBoolean();
 793             break;
 794         case IsNumber:
 795             compileIsNumber();
 796             break;
 797         case IsString:
 798             compileIsString();
 799             break;
 800         case IsObject:
 801             compileIsObject();
 802             break;
 803         case IsObjectOrNull:
 804             compileIsObjectOrNull();
 805             break;
 806         case IsFunction:
 807             compileIsFunction();
 808             break;
 809         case TypeOf:
 810             compileTypeOf();
 811             break;
 812         case CheckHasInstance:
 813             compileCheckHasInstance();
 814             break;
 815         case InstanceOf:
 816             compileInstanceOf();
 817             break;
 818         case CountExecution:
 819             compileCountExecution();
 820             break;
 821         case StoreBarrier:
 822             compileStoreBarrier();
 823             break;
 824         case HasIndexedProperty:
 825             compileHasIndexedProperty();
 826             break;
 827         case HasGenericProperty:
 828             compileHasGenericProperty();
 829             break;
 830         case HasStructureProperty:
 831             compileHasStructureProperty();
 832             break;
 833         case GetDirectPname:
 834             compileGetDirectPname();
 835             break;
 836         case GetEnumerableLength:
 837             compileGetEnumerableLength();
 838             break;
 839         case GetPropertyEnumerator:
 840             compileGetPropertyEnumerator();
 841             break;
 842         case GetEnumeratorStructurePname:
 843             compileGetEnumeratorStructurePname();
 844             break;
 845         case GetEnumeratorGenericPname:
 846             compileGetEnumeratorGenericPname();
 847             break;
 848         case ToIndexString:
 849             compileToIndexString();
 850             break;
 851         case CheckStructureImmediate:
 852             compileCheckStructureImmediate();
 853             break;
 854         case MaterializeNewObject:
 855             compileMaterializeNewObject();
 856             break;
 857         case MaterializeCreateActivation:
 858             compileMaterializeCreateActivation();
 859             break;
 860
 861         case PhantomLocal:
 862         case LoopHint:
 863         case MovHint:
 864         case ZombieHint:
 865         case PhantomNewObject:
 866         case PhantomNewFunction:
 867         case PhantomCreateActivation:
 868         case PhantomDirectArguments:
 869         case PhantomClonedArguments:
 870         case PutHint:
 871         case BottomValue:
 872         case KillStack:
 873             break;
 874         default:
 875             DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
 876             break;
 877         }
 878
 879         if (m_node->isTerminal())
 880             return false;
 881
 882         if (!m_state.isValid()) {
 883             safelyInvalidateAfterTermination();
 884             return false;
 885         }
 886
 887         m_availabilityCalculator.executeNode(m_node);
 888         m_interpreter.executeEffects(nodeIndex);
 889
 890         return true;
 891     }
 892
 893     void compileUpsilon()
 894     {
 895         LValue destination = m_phis.get(m_node->phi());
 896
 897         switch (m_node->child1().useKind()) {
 898         case DoubleRepUse:
 899             m_out.set(lowDouble(m_node->child1()), destination);
 900             break;
 901         case Int32Use:
 902             m_out.set(lowInt32(m_node->child1()), destination);
 903             break;
 904         case Int52RepUse:
 905             m_out.set(lowInt52(m_node->child1()), destination);
 906             break;
 907         case BooleanUse:
 908             m_out.set(lowBoolean(m_node->child1()), destination);
 909             break;
 910         case CellUse:
 911             m_out.set(lowCell(m_node->child1()), destination);
 912             break;
 913         case UntypedUse:
 914             m_out.set(lowJSValue(m_node->child1()), destination);
 915             break;
 916         default:
 917             DFG_CRASH(m_graph, m_node, "Bad use kind");
 918             break;
 919         }
 920     }
 921
 922     void compilePhi()
 923     {
 924         LValue source = m_phis.get(m_node);
 925
 926         switch (m_node->flags() & NodeResultMask) {
 927         case NodeResultDouble:
 928             setDouble(m_out.get(source));
 929             break;
 930         case NodeResultInt32:
 931             setInt32(m_out.get(source));
 932             break;
 933         case NodeResultInt52:
 934             setInt52(m_out.get(source));
 935             break;
 936         case NodeResultBoolean:
 937             setBoolean(m_out.get(source));
 938             break;
 939         case NodeResultJS:
 940             setJSValue(m_out.get(source));
 941             break;
 942         default:
 943             DFG_CRASH(m_graph, m_node, "Bad use kind");
 944             break;
 945         }
 946     }
 947
 948     void compileDoubleConstant()
 949     {
 950         setDouble(m_out.constDouble(m_node->asNumber()));
 951     }
 952
 953     void compileInt52Constant()
 954     {
 955         int64_t value = m_node->asMachineInt();
 956
 957         setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
 958         setStrictInt52(m_out.constInt64(value));
 959     }
 960
 961     void compileDoubleRep()
 962     {
 963         switch (m_node->child1().useKind()) {
 964         case RealNumberUse: {
 965             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
 966
 967             LValue doubleValue = unboxDouble(value);
 968
 969             LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("DoubleRep RealNumberUse int case"));
 970             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("DoubleRep continuation"));
 971
 972             ValueFromBlock fastResult = m_out.anchor(doubleValue);
 973             m_out.branch(
 974                 m_out.doubleEqual(doubleValue, doubleValue),
 975                 usually(continuation), rarely(intCase));
 976
 977             LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
 978
 979             FTL_TYPE_CHECK(
 980                 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
 981                 isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble));
 982             ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value)));
 983             m_out.jump(continuation);
 984
 985             m_out.appendTo(continuation, lastNext);
 986
 987             setDouble(m_out.phi(m_out.doubleType, fastResult, slowResult));
 988             return;
 989         }
 990
 991         case NotCellUse:
 992         case NumberUse: {
 993             bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse;
 994
 995             LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
 996
 997             LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing int case"));
 998             LBasicBlock doubleTesting = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing double case"));
 999             LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing double case"));
1000             LBasicBlock nonDoubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing undefined case"));
1001             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing continuation"));
1002
1003             m_out.branch(
1004                 isNotInt32(value, provenType(m_node->child1())),
1005                 unsure(doubleTesting), unsure(intCase));
1006
1007             LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting);
1008
1009             ValueFromBlock intToDouble = m_out.anchor(
1010                 m_out.intToDouble(unboxInt32(value)));
1011             m_out.jump(continuation);
1012
1013             m_out.appendTo(doubleTesting, doubleCase);
1014             LValue valueIsNumber = isNumber(value, provenType(m_node->child1()));
1015             m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase));
1016
1017             m_out.appendTo(doubleCase, nonDoubleCase);
1018             ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value));
1019             m_out.jump(continuation);
1020
1021             if (shouldConvertNonNumber) {
1022                 LBasicBlock undefinedCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble converting undefined case"));
1023                 LBasicBlock testNullCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing null case"));
1024                 LBasicBlock nullCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble converting null case"));
1025                 LBasicBlock testBooleanTrueCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing boolean true case"));
1026                 LBasicBlock convertBooleanTrueCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble convert boolean true case"));
1027                 LBasicBlock convertBooleanFalseCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble convert boolean false case"));
1028
1029                 m_out.appendTo(nonDoubleCase, undefinedCase);
1030                 LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(ValueUndefined));
1031                 m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase));
1032
1033                 m_out.appendTo(undefinedCase, testNullCase);
1034                 ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN));
1035                 m_out.jump(continuation);
1036
1037                 m_out.appendTo(testNullCase, nullCase);
1038                 LValue valueIsNull = m_out.equal(value, m_out.constInt64(ValueNull));
1039                 m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase));
1040
1041                 m_out.appendTo(nullCase, testBooleanTrueCase);
1042                 ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0));
1043                 m_out.jump(continuation);
1044
1045                 m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase);
1046                 LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(ValueTrue));
1047                 m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase));
1048
1049                 m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase);
1050                 ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1));
1051                 m_out.jump(continuation);
1052
1053                 m_out.appendTo(convertBooleanFalseCase, continuation);
1054
1055                 LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(ValueFalse));
1056                 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCell, valueIsNotBooleanFalse);
1057                 ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0));
1058                 m_out.jump(continuation);
1059
1060                 m_out.appendTo(continuation, lastNext);
1061                 setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse));
1062                 return;
1063             }
1064             m_out.appendTo(nonDoubleCase, continuation);
1065             FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue);
1066             m_out.unreachable();
1067
1068             m_out.appendTo(continuation, lastNext);
1069
1070             setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble));
1071             return;
1072         }
1073
1074         case Int52RepUse: {
1075             setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
1076             return;
1077         }
1078
1079         default:
1080             DFG_CRASH(m_graph, m_node, "Bad use kind");
1081         }
1082     }
1083
1084     void compileDoubleAsInt32()
1085     {
1086         LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode()));
1087         setInt32(integerValue);
1088     }
1089
1090     void compileValueRep()
1091     {
1092         switch (m_node->child1().useKind()) {
1093         case DoubleRepUse: {
1094             LValue value = lowDouble(m_node->child1());
1095
1096             if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
1097                 value = m_out.select(
1098                     m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
1099             }
1100
1101             setJSValue(boxDouble(value));
1102             return;
1103         }
1104
1105         case Int52RepUse: {
1106             setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
1107             return;
1108         }
1109
1110         default:
1111             DFG_CRASH(m_graph, m_node, "Bad use kind");
1112         }
1113     }
1114
1115     void compileInt52Rep()
1116     {
1117         switch (m_node->child1().useKind()) {
1118         case Int32Use:
1119             setStrictInt52(m_out.signExt(lowInt32(m_node->child1()), m_out.int64));
1120             return;
1121
1122         case MachineIntUse:
1123             setStrictInt52(
1124                 jsValueToStrictInt52(
1125                     m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1126             return;
1127
1128         case DoubleRepMachineIntUse:
1129             setStrictInt52(
1130                 doubleToStrictInt52(
1131                     m_node->child1(), lowDouble(m_node->child1())));
1132             return;
1133
1134         default:
1135             RELEASE_ASSERT_NOT_REACHED();
1136         }
1137     }
1138
1139     void compileValueToInt32()
1140     {
1141         switch (m_node->child1().useKind()) {
1142         case Int52RepUse:
1143             setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1144             break;
1145
1146         case DoubleRepUse:
1147             setInt32(doubleToInt32(lowDouble(m_node->child1())));
1148             break;
1149
1150         case NumberUse:
1151         case NotCellUse: {
1152             LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1153             if (isValid(value)) {
1154                 setInt32(value.value());
1155                 break;
1156             }
1157
1158             value = m_jsValueValues.get(m_node->child1().node());
1159             if (isValid(value)) {
1160                 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1161                 break;
1162             }
1163
1164             // We'll basically just get here for constants. But it's good to have this
1165             // catch-all since we often add new representations into the mix.
1166             setInt32(
1167                 numberOrNotCellToInt32(
1168                     m_node->child1(),
1169                     lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1170             break;
1171         }
1172
1173         default:
1174             DFG_CRASH(m_graph, m_node, "Bad use kind");
1175             break;
1176         }
1177     }
1178
1179     void compileBooleanToNumber()
1180     {
1181         switch (m_node->child1().useKind()) {
1182         case BooleanUse: {
1183             setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32));
1184             return;
1185         }
1186
1187         case UntypedUse: {
1188             LValue value = lowJSValue(m_node->child1());
1189
1190             if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) {
1191                 setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One));
1192                 return;
1193             }
1194
1195             LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("BooleanToNumber boolean case"));
1196             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("BooleanToNumber continuation"));
1197
1198             ValueFromBlock notBooleanResult = m_out.anchor(value);
1199             m_out.branch(
1200                 isBoolean(value, provenType(m_node->child1())),
1201                 unsure(booleanCase), unsure(continuation));
1202
1203             LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1204             ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1205                 m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber));
1206             m_out.jump(continuation);
1207
1208             m_out.appendTo(continuation, lastNext);
1209             setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult));
1210             return;
1211         }
1212
1213         default:
1214             RELEASE_ASSERT_NOT_REACHED();
1215             return;
1216         }
1217     }
1218
1219     void compileExtractOSREntryLocal()
1220     {
1221         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1222             m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1223         setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1224     }
1225
1226     void compileGetStack()
1227     {
1228         // GetLocals arise only for captured variables and arguments. For arguments, we might have
1229         // already loaded it.
1230         if (LValue value = m_loadedArgumentValues.get(m_node)) {
1231             setJSValue(value);
1232             return;
1233         }
1234
1235         StackAccessData* data = m_node->stackAccessData();
1236         AbstractValue& value = m_state.variables().operand(data->local);
1237
1238         DFG_ASSERT(m_graph, m_node, isConcrete(data->format));
1239         DFG_ASSERT(m_graph, m_node, data->format != FlushedDouble); // This just happens to not arise for GetStacks, right now. It would be trivial to support.
1240
1241         if (isInt32Speculation(value.m_type))
1242             setInt32(m_out.load32(payloadFor(data->machineLocal)));
1243         else
1244             setJSValue(m_out.load64(addressFor(data->machineLocal)));
1245     }
1246
1247     void compilePutStack()
1248     {
1249         StackAccessData* data = m_node->stackAccessData();
1250         switch (data->format) {
1251         case FlushedJSValue: {
1252             LValue value = lowJSValue(m_node->child1());
1253             m_out.store64(value, addressFor(data->machineLocal));
1254             break;
1255         }
1256
1257         case FlushedDouble: {
1258             LValue value = lowDouble(m_node->child1());
1259             m_out.storeDouble(value, addressFor(data->machineLocal));
1260             break;
1261         }
1262
1263         case FlushedInt32: {
1264             LValue value = lowInt32(m_node->child1());
1265             m_out.store32(value, payloadFor(data->machineLocal));
1266             break;
1267         }
1268
1269         case FlushedInt52: {
1270             LValue value = lowInt52(m_node->child1());
1271             m_out.store64(value, addressFor(data->machineLocal));
1272             break;
1273         }
1274
1275         case FlushedCell: {
1276             LValue value = lowCell(m_node->child1());
1277             m_out.store64(value, addressFor(data->machineLocal));
1278             break;
1279         }
1280
1281         case FlushedBoolean: {
1282             speculateBoolean(m_node->child1());
1283             m_out.store64(
1284                 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1285                 addressFor(data->machineLocal));
1286             break;
1287         }
1288
1289         default:
1290             DFG_CRASH(m_graph, m_node, "Bad flush format");
1291             break;
1292         }
1293     }
1294
1295     void compileNoOp()
1296     {
1297         DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
1298     }
1299
1300     void compileToThis()
1301     {
1302         LValue value = lowJSValue(m_node->child1());
1303
1304         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToThis is cell case"));
1305         LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ToThis slow case"));
1306         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToThis continuation"));
1307
1308         m_out.branch(
1309             isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
1310
1311         LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1312         ValueFromBlock fastResult = m_out.anchor(value);
1313         m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase));
1314
1315         m_out.appendTo(slowCase, continuation);
1316         J_JITOperation_EJ function;
1317         if (m_graph.isStrictModeFor(m_node->origin.semantic))
1318             function = operationToThisStrict;
1319         else
1320             function = operationToThis;
1321         ValueFromBlock slowResult = m_out.anchor(
1322             vmCall(m_out.operation(function), m_callFrame, value));
1323         m_out.jump(continuation);
1324
1325         m_out.appendTo(continuation, lastNext);
1326         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1327     }
1328
1329     void compileValueAdd()
1330     {
1331         J_JITOperation_EJJ operation;
1332         if (!(provenType(m_node->child1()) & SpecFullNumber)
1333             && !(provenType(m_node->child2()) & SpecFullNumber))
1334             operation = operationValueAddNotNumber;
1335         else
1336             operation = operationValueAdd;
1337         setJSValue(vmCall(
1338             m_out.operation(operation), m_callFrame,
1339             lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
1340     }
1341
1342     void compileArithAddOrSub()
1343     {
1344         bool isSub =  m_node->op() == ArithSub;
1345         switch (m_node->binaryUseKind()) {
1346         case Int32Use: {
1347             LValue left = lowInt32(m_node->child1());
1348             LValue right = lowInt32(m_node->child2());
1349
1350             if (!shouldCheckOverflow(m_node->arithMode())) {
1351                 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
1352                 break;
1353             }
1354
1355             LValue result;
1356             if (!isSub) {
1357                 result = m_out.addWithOverflow32(left, right);
1358
1359                 if (doesKill(m_node->child2())) {
1360                     addAvailableRecovery(
1361                         m_node->child2(), SubRecovery,
1362                         m_out.extractValue(result, 0), left, ValueFormatInt32);
1363                 } else if (doesKill(m_node->child1())) {
1364                     addAvailableRecovery(
1365                         m_node->child1(), SubRecovery,
1366                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1367                 }
1368             } else {
1369                 result = m_out.subWithOverflow32(left, right);
1370
1371                 if (doesKill(m_node->child2())) {
1372                     // result = left - right
1373                     // result - left = -right
1374                     // right = left - result
1375                     addAvailableRecovery(
1376                         m_node->child2(), SubRecovery,
1377                         left, m_out.extractValue(result, 0), ValueFormatInt32);
1378                 } else if (doesKill(m_node->child1())) {
1379                     // result = left - right
1380                     // result + right = left
1381                     addAvailableRecovery(
1382                         m_node->child1(), AddRecovery,
1383                         m_out.extractValue(result, 0), right, ValueFormatInt32);
1384                 }
1385             }
1386
1387             speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
1388             setInt32(m_out.extractValue(result, 0));
1389             break;
1390         }
1391
1392         case Int52RepUse: {
1393             if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)
1394                 && !abstractValue(m_node->child2()).couldBeType(SpecInt52)) {
1395                 Int52Kind kind;
1396                 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1397                 LValue right = lowInt52(m_node->child2(), kind);
1398                 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
1399                 break;
1400             }
1401
1402             LValue left = lowInt52(m_node->child1());
1403             LValue right = lowInt52(m_node->child2());
1404
1405             LValue result;
1406             if (!isSub) {
1407                 result = m_out.addWithOverflow64(left, right);
1408
1409                 if (doesKill(m_node->child2())) {
1410                     addAvailableRecovery(
1411                         m_node->child2(), SubRecovery,
1412                         m_out.extractValue(result, 0), left, ValueFormatInt52);
1413                 } else if (doesKill(m_node->child1())) {
1414                     addAvailableRecovery(
1415                         m_node->child1(), SubRecovery,
1416                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1417                 }
1418             } else {
1419                 result = m_out.subWithOverflow64(left, right);
1420
1421                 if (doesKill(m_node->child2())) {
1422                     // result = left - right
1423                     // result - left = -right
1424                     // right = left - result
1425                     addAvailableRecovery(
1426                         m_node->child2(), SubRecovery,
1427                         left, m_out.extractValue(result, 0), ValueFormatInt52);
1428                 } else if (doesKill(m_node->child1())) {
1429                     // result = left - right
1430                     // result + right = left
1431                     addAvailableRecovery(
1432                         m_node->child1(), AddRecovery,
1433                         m_out.extractValue(result, 0), right, ValueFormatInt52);
1434                 }
1435             }
1436
1437             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1));
1438             setInt52(m_out.extractValue(result, 0));
1439             break;
1440         }
1441
1442         case DoubleRepUse: {
1443             LValue C1 = lowDouble(m_node->child1());
1444             LValue C2 = lowDouble(m_node->child2());
1445
1446             setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
1447             break;
1448         }
1449
1450         default:
1451             DFG_CRASH(m_graph, m_node, "Bad use kind");
1452             break;
1453         }
1454     }
1455
1456     void compileArithClz32()
1457     {
1458         LValue operand = lowInt32(m_node->child1());
1459         LValue isZeroUndef = m_out.booleanFalse;
1460         setInt32(m_out.ctlz32(operand, isZeroUndef));
1461     }
1462
1463     void compileArithMul()
1464     {
1465         switch (m_node->binaryUseKind()) {
1466         case Int32Use: {
1467             LValue left = lowInt32(m_node->child1());
1468             LValue right = lowInt32(m_node->child2());
1469
1470             LValue result;
1471
1472             if (!shouldCheckOverflow(m_node->arithMode()))
1473                 result = m_out.mul(left, right);
1474             else {
1475                 LValue overflowResult = m_out.mulWithOverflow32(left, right);
1476                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1477                 result = m_out.extractValue(overflowResult, 0);
1478             }
1479
1480             if (shouldCheckNegativeZero(m_node->arithMode())) {
1481                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1482                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1483
1484                 m_out.branch(
1485                     m_out.notZero32(result), usually(continuation), rarely(slowCase));
1486
1487                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1488                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int32Zero), m_out.lessThan(right, m_out.int32Zero));
1489                 speculate(NegativeZero, noValue(), 0, cond);
1490                 m_out.jump(continuation);
1491                 m_out.appendTo(continuation, lastNext);
1492             }
1493
1494             setInt32(result);
1495             break;
1496         }
1497
1498         case Int52RepUse: {
1499             Int52Kind kind;
1500             LValue left = lowWhicheverInt52(m_node->child1(), kind);
1501             LValue right = lowInt52(m_node->child2(), opposite(kind));
1502
1503             LValue overflowResult = m_out.mulWithOverflow64(left, right);
1504             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1505             LValue result = m_out.extractValue(overflowResult, 0);
1506
1507             if (shouldCheckNegativeZero(m_node->arithMode())) {
1508                 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1509                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1510
1511                 m_out.branch(
1512                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
1513
1514                 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1515                 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int64Zero), m_out.lessThan(right, m_out.int64Zero));
1516                 speculate(NegativeZero, noValue(), 0, cond);
1517                 m_out.jump(continuation);
1518                 m_out.appendTo(continuation, lastNext);
1519             }
1520
1521             setInt52(result);
1522             break;
1523         }
1524
1525         case DoubleRepUse: {
1526             setDouble(
1527                 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1528             break;
1529         }
1530
1531         default:
1532             DFG_CRASH(m_graph, m_node, "Bad use kind");
1533             break;
1534         }
1535     }
1536
1537     void compileArithDiv()
1538     {
1539         switch (m_node->binaryUseKind()) {
1540         case Int32Use: {
1541             LValue numerator = lowInt32(m_node->child1());
1542             LValue denominator = lowInt32(m_node->child2());
1543
1544             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
1545             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
1546             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
1547
1548             Vector<ValueFromBlock, 3> results;
1549
1550             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1551
1552             m_out.branch(
1553                 m_out.above(adjustedDenominator, m_out.int32One),
1554                 usually(continuation), rarely(unsafeDenominator));
1555
1556             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1557
1558             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1559
1560             if (shouldCheckOverflow(m_node->arithMode())) {
1561                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1562                 speculate(Overflow, noValue(), 0, cond);
1563                 m_out.jump(continuation);
1564             } else {
1565                 // This is the case where we convert the result to an int after we're done. So,
1566                 // if the denominator is zero, then the result should be zero.
1567                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1568                 // check above) and the numerator is -2^31 then the result should be -2^31.
1569
1570                 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
1571                 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
1572                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
1573
1574                 m_out.branch(
1575                     m_out.isZero32(denominator), rarely(divByZero), usually(notDivByZero));
1576
1577                 m_out.appendTo(divByZero, notDivByZero);
1578                 results.append(m_out.anchor(m_out.int32Zero));
1579                 m_out.jump(done);
1580
1581                 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
1582                 m_out.branch(
1583                     m_out.equal(numerator, neg2ToThe31),
1584                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1585
1586                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1587                 results.append(m_out.anchor(neg2ToThe31));
1588                 m_out.jump(done);
1589             }
1590
1591             m_out.appendTo(continuation, done);
1592
1593             if (shouldCheckNegativeZero(m_node->arithMode())) {
1594                 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
1595                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
1596
1597                 m_out.branch(
1598                     m_out.isZero32(numerator),
1599                     rarely(zeroNumerator), usually(numeratorContinuation));
1600
1601                 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
1602
1603                 speculate(
1604                     NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
1605
1606                 m_out.jump(numeratorContinuation);
1607
1608                 m_out.appendTo(numeratorContinuation, innerLastNext);
1609             }
1610
1611             LValue result = m_out.div(numerator, denominator);
1612
1613             if (shouldCheckOverflow(m_node->arithMode())) {
1614                 speculate(
1615                     Overflow, noValue(), 0,
1616                     m_out.notEqual(m_out.mul(result, denominator), numerator));
1617             }
1618
1619             results.append(m_out.anchor(result));
1620             m_out.jump(done);
1621
1622             m_out.appendTo(done, lastNext);
1623
1624             setInt32(m_out.phi(m_out.int32, results));
1625             break;
1626         }
1627
1628         case DoubleRepUse: {
1629             setDouble(m_out.doubleDiv(
1630                 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1631             break;
1632         }
1633
1634         default:
1635             DFG_CRASH(m_graph, m_node, "Bad use kind");
1636             break;
1637         }
1638     }
1639
1640     void compileArithMod()
1641     {
1642         switch (m_node->binaryUseKind()) {
1643         case Int32Use: {
1644             LValue numerator = lowInt32(m_node->child1());
1645             LValue denominator = lowInt32(m_node->child2());
1646
1647             LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
1648             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
1649             LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
1650
1651             Vector<ValueFromBlock, 3> results;
1652
1653             LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1654
1655             m_out.branch(
1656                 m_out.above(adjustedDenominator, m_out.int32One),
1657                 usually(continuation), rarely(unsafeDenominator));
1658
1659             LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1660
1661             LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1662
1663             // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
1664             // separate case for that. But it probably doesn't matter so much.
1665             if (shouldCheckOverflow(m_node->arithMode())) {
1666                 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1667                 speculate(Overflow, noValue(), 0, cond);
1668                 m_out.jump(continuation);
1669             } else {
1670                 // This is the case where we convert the result to an int after we're done. So,
1671                 // if the denominator is zero, then the result should be result should be zero.
1672                 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1673                 // check above) and the numerator is -2^31 then the result should be -2^31.
1674
1675                 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
1676                 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
1677                 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
1678
1679                 m_out.branch(
1680                     m_out.isZero32(denominator), rarely(modByZero), usually(notModByZero));
1681
1682                 m_out.appendTo(modByZero, notModByZero);
1683                 results.append(m_out.anchor(m_out.int32Zero));
1684                 m_out.jump(done);
1685
1686                 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
1687                 m_out.branch(
1688                     m_out.equal(numerator, neg2ToThe31),
1689                     rarely(neg2ToThe31ByNeg1), usually(continuation));
1690
1691                 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1692                 results.append(m_out.anchor(m_out.int32Zero));
1693                 m_out.jump(done);
1694             }
1695
1696             m_out.appendTo(continuation, done);
1697
1698             LValue remainder = m_out.rem(numerator, denominator);
1699
1700             if (shouldCheckNegativeZero(m_node->arithMode())) {
1701                 LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator"));
1702                 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
1703
1704                 m_out.branch(
1705                     m_out.lessThan(numerator, m_out.int32Zero),
1706                     unsure(negativeNumerator), unsure(numeratorContinuation));
1707
1708                 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
1709
1710                 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
1711
1712                 m_out.jump(numeratorContinuation);
1713
1714                 m_out.appendTo(numeratorContinuation, innerLastNext);
1715             }
1716
1717             results.append(m_out.anchor(remainder));
1718             m_out.jump(done);
1719
1720             m_out.appendTo(done, lastNext);
1721
1722             setInt32(m_out.phi(m_out.int32, results));
1723             break;
1724         }
1725
1726         case DoubleRepUse: {
1727             setDouble(
1728                 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1729             break;
1730         }
1731
1732         default:
1733             DFG_CRASH(m_graph, m_node, "Bad use kind");
1734             break;
1735         }
1736     }
1737
1738     void compileArithMinOrMax()
1739     {
1740         switch (m_node->binaryUseKind()) {
1741         case Int32Use: {
1742             LValue left = lowInt32(m_node->child1());
1743             LValue right = lowInt32(m_node->child2());
1744
1745             setInt32(
1746                 m_out.select(
1747                     m_node->op() == ArithMin
1748                         ? m_out.lessThan(left, right)
1749                         : m_out.lessThan(right, left),
1750                     left, right));
1751             break;
1752         }
1753
1754         case DoubleRepUse: {
1755             LValue left = lowDouble(m_node->child1());
1756             LValue right = lowDouble(m_node->child2());
1757
1758             LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
1759             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
1760
1761             Vector<ValueFromBlock, 2> results;
1762
1763             results.append(m_out.anchor(left));
1764             m_out.branch(
1765                 m_node->op() == ArithMin
1766                     ? m_out.doubleLessThan(left, right)
1767                     : m_out.doubleGreaterThan(left, right),
1768                 unsure(continuation), unsure(notLessThan));
1769
1770             LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1771             results.append(m_out.anchor(m_out.select(
1772                 m_node->op() == ArithMin
1773                     ? m_out.doubleGreaterThanOrEqual(left, right)
1774                     : m_out.doubleLessThanOrEqual(left, right),
1775                 right, m_out.constDouble(PNaN))));
1776             m_out.jump(continuation);
1777
1778             m_out.appendTo(continuation, lastNext);
1779             setDouble(m_out.phi(m_out.doubleType, results));
1780             break;
1781         }
1782
1783         default:
1784             DFG_CRASH(m_graph, m_node, "Bad use kind");
1785             break;
1786         }
1787     }
1788
1789     void compileArithAbs()
1790     {
1791         switch (m_node->child1().useKind()) {
1792         case Int32Use: {
1793             LValue value = lowInt32(m_node->child1());
1794
1795             LValue mask = m_out.aShr(value, m_out.constInt32(31));
1796             LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1797
1798             speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
1799
1800             setInt32(result);
1801             break;
1802         }
1803
1804         case DoubleRepUse: {
1805             setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1806             break;
1807         }
1808
1809         default:
1810             DFG_CRASH(m_graph, m_node, "Bad use kind");
1811             break;
1812         }
1813     }
1814
1815     void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); }
1816
1817     void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); }
1818
1819     void compileArithPow()
1820     {
1821         // FIXME: investigate llvm.powi to better understand its performance characteristics.
1822         // It might be better to have the inline loop in DFG too.
1823         if (m_node->child2().useKind() == Int32Use)
1824             setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
1825         else {
1826             LValue base = lowDouble(m_node->child1());
1827             LValue exponent = lowDouble(m_node->child2());
1828
1829             LBasicBlock integerExponentIsSmallBlock = FTL_NEW_BLOCK(m_out, ("ArithPow test integer exponent is small."));
1830             LBasicBlock integerExponentPowBlock = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, (int)double)."));
1831             LBasicBlock doubleExponentPowBlockEntry = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, double)."));
1832             LBasicBlock nanExceptionExponentIsInfinity = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check exponent is infinity."));
1833             LBasicBlock nanExceptionBaseIsOne = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check base is one."));
1834             LBasicBlock powBlock = FTL_NEW_BLOCK(m_out, ("ArithPow regular pow"));
1835             LBasicBlock nanExceptionResultIsNaN = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, result is NaN."));
1836             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithPow continuation"));
1837
1838             LValue integerExponent = m_out.fpToInt32(exponent);
1839             LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
1840             LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
1841             m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
1842
1843             LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
1844             LValue integerExponentBelow1000 = m_out.below(integerExponent, m_out.constInt32(1000));
1845             m_out.branch(integerExponentBelow1000, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
1846
1847             m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
1848             ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
1849             m_out.jump(continuation);
1850
1851             // If y is NaN, the result is NaN.
1852             m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionExponentIsInfinity);
1853             LValue exponentIsNaN;
1854             if (provenType(m_node->child2()) & SpecDoubleNaN)
1855                 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
1856             else
1857                 exponentIsNaN = m_out.booleanFalse;
1858             m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionExponentIsInfinity));
1859
1860             // If abs(x) is 1 and y is +infinity, the result is NaN.
1861             // If abs(x) is 1 and y is -infinity, the result is NaN.
1862             m_out.appendTo(nanExceptionExponentIsInfinity, nanExceptionBaseIsOne);
1863             LValue absoluteExponent = m_out.doubleAbs(exponent);
1864             LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
1865             m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionBaseIsOne), usually(powBlock));
1866
1867             m_out.appendTo(nanExceptionBaseIsOne, powBlock);
1868             LValue absoluteBase = m_out.doubleAbs(base);
1869             LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
1870             m_out.branch(absoluteBaseIsOne, unsure(nanExceptionResultIsNaN), unsure(powBlock));
1871
1872             m_out.appendTo(powBlock, nanExceptionResultIsNaN);
1873             ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
1874             m_out.jump(continuation);
1875
1876             m_out.appendTo(nanExceptionResultIsNaN, continuation);
1877             ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
1878             m_out.jump(continuation);
1879
1880             m_out.appendTo(continuation, lastNext);
1881             setDouble(m_out.phi(m_out.doubleType, powDoubleIntResult, powResult, pureNan));
1882         }
1883     }
1884
1885     void compileArithRound()
1886     {
1887         LBasicBlock realPartIsMoreThanHalf = FTL_NEW_BLOCK(m_out, ("ArithRound should round down"));
1888         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithRound continuation"));
1889
1890         LValue value = lowDouble(m_node->child1());
1891         LValue integerValue = m_out.ceil64(value);
1892         ValueFromBlock integerValueResult = m_out.anchor(integerValue);
1893
1894         LValue realPart = m_out.doubleSub(integerValue, value);
1895
1896         m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation));
1897
1898         LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation);
1899         LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1));
1900         ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown);
1901         m_out.jump(continuation);
1902         m_out.appendTo(continuation, lastNext);
1903
1904         LValue result = m_out.phi(m_out.doubleType, integerValueResult, integerValueRoundedDownResult);
1905
1906         if (producesInteger(m_node->arithRoundingMode())) {
1907             LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()));
1908             setInt32(integerValue);
1909         } else
1910             setDouble(result);
1911     }
1912
1913     void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); }
1914
1915     void compileArithLog() { setDouble(m_out.doubleLog(lowDouble(m_node->child1()))); }
1916
1917     void compileArithFRound()
1918     {
1919         LValue floatValue = m_out.fpCast(lowDouble(m_node->child1()), m_out.floatType);
1920         setDouble(m_out.fpCast(floatValue, m_out.doubleType));
1921     }
1922
1923     void compileArithNegate()
1924     {
1925         switch (m_node->child1().useKind()) {
1926         case Int32Use: {
1927             LValue value = lowInt32(m_node->child1());
1928
1929             LValue result;
1930             if (!shouldCheckOverflow(m_node->arithMode()))
1931                 result = m_out.neg(value);
1932             else if (!shouldCheckNegativeZero(m_node->arithMode())) {
1933                 // We don't have a negate-with-overflow intrinsic. Hopefully this
1934                 // does the trick, though.
1935                 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
1936                 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1937                 result = m_out.extractValue(overflowResult, 0);
1938             } else {
1939                 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
1940                 result = m_out.neg(value);
1941             }
1942
1943             setInt32(result);
1944             break;
1945         }
1946
1947         case Int52RepUse: {
1948             if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)) {
1949                 Int52Kind kind;
1950                 LValue value = lowWhicheverInt52(m_node->child1(), kind);
1951                 LValue result = m_out.neg(value);
1952                 if (shouldCheckNegativeZero(m_node->arithMode()))
1953                     speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1954                 setInt52(result, kind);
1955                 break;
1956             }
1957
1958             LValue value = lowInt52(m_node->child1());
1959             LValue overflowResult = m_out.subWithOverflow64(m_out.int64Zero, value);
1960             speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1961             LValue result = m_out.extractValue(overflowResult, 0);
1962             speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1963             setInt52(result);
1964             break;
1965         }
1966
1967         case DoubleRepUse: {
1968             setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
1969             break;
1970         }
1971
1972         default:
1973             DFG_CRASH(m_graph, m_node, "Bad use kind");
1974             break;
1975         }
1976     }
1977
1978     void compileBitAnd()
1979     {
1980         setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1981     }
1982
1983     void compileBitOr()
1984     {
1985         setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1986     }
1987
1988     void compileBitXor()
1989     {
1990         setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1991     }
1992
1993     void compileBitRShift()
1994     {
1995         setInt32(m_out.aShr(
1996             lowInt32(m_node->child1()),
1997             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1998     }
1999
2000     void compileBitLShift()
2001     {
2002         setInt32(m_out.shl(
2003             lowInt32(m_node->child1()),
2004             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2005     }
2006
2007     void compileBitURShift()
2008     {
2009         setInt32(m_out.lShr(
2010             lowInt32(m_node->child1()),
2011             m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2012     }
2013
2014     void compileUInt32ToNumber()
2015     {
2016         LValue value = lowInt32(m_node->child1());
2017
2018         if (doesOverflow(m_node->arithMode())) {
2019             setDouble(m_out.unsignedToDouble(value));
2020             return;
2021         }
2022
2023         speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
2024         setInt32(value);
2025     }
2026
2027     void compileCheckStructure()
2028     {
2029         LValue cell = lowCell(m_node->child1());
2030
2031         ExitKind exitKind;
2032         if (m_node->child1()->hasConstant())
2033             exitKind = BadConstantCache;
2034         else
2035             exitKind = BadCache;
2036
2037         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2038
2039         checkStructure(
2040             structureID, jsValueValue(cell), exitKind, m_node->structureSet(),
2041             [this] (Structure* structure) {
2042                 return weakStructureID(structure);
2043             });
2044     }
2045
2046     void compileCheckCell()
2047     {
2048         LValue cell = lowCell(m_node->child1());
2049
2050         speculate(
2051             BadCell, jsValueValue(cell), m_node->child1().node(),
2052             m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
2053     }
2054
2055     void compileCheckBadCell()
2056     {
2057         terminate(BadCell);
2058     }
2059
2060     void compileCheckNotEmpty()
2061     {
2062         speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
2063     }
2064
2065     void compileGetExecutable()
2066     {
2067         LValue cell = lowCell(m_node->child1());
2068         speculateFunction(m_node->child1(), cell);
2069         setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
2070     }
2071
2072     void compileArrayifyToStructure()
2073     {
2074         LValue cell = lowCell(m_node->child1());
2075         LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
2076
2077         LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
2078         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
2079
2080         LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2081
2082         m_out.branch(
2083             m_out.notEqual(structureID, weakStructureID(m_node->structure())),
2084             rarely(unexpectedStructure), usually(continuation));
2085
2086         LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
2087
2088         if (property) {
2089             switch (m_node->arrayMode().type()) {
2090             case Array::Int32:
2091             case Array::Double:
2092             case Array::Contiguous:
2093                 speculate(
2094                     Uncountable, noValue(), 0,
2095                     m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
2096                 break;
2097             default:
2098                 break;
2099             }
2100         }
2101
2102         switch (m_node->arrayMode().type()) {
2103         case Array::Int32:
2104             vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
2105             break;
2106         case Array::Double:
2107             vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
2108             break;
2109         case Array::Contiguous:
2110             vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
2111             break;
2112         case Array::ArrayStorage:
2113         case Array::SlowPutArrayStorage:
2114             vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
2115             break;
2116         default:
2117             DFG_CRASH(m_graph, m_node, "Bad array type");
2118             break;
2119         }
2120
2121         structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2122         speculate(
2123             BadIndexingType, jsValueValue(cell), 0,
2124             m_out.notEqual(structureID, weakStructureID(m_node->structure())));
2125         m_out.jump(continuation);
2126
2127         m_out.appendTo(continuation, lastNext);
2128     }
2129
2130     void compilePutStructure()
2131     {
2132         m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
2133
2134         Structure* oldStructure = m_node->transition()->previous;
2135         Structure* newStructure = m_node->transition()->next;
2136         ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
2137         ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
2138         ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
2139
2140         LValue cell = lowCell(m_node->child1());
2141         m_out.store32(
2142             weakStructureID(newStructure),
2143             cell, m_heaps.JSCell_structureID);
2144     }
2145
2146     void compileGetById()
2147     {
2148         // Pretty much the only reason why we don't also support GetByIdFlush is because:
2149         // https://bugs.webkit.org/show_bug.cgi?id=125711
2150
2151         switch (m_node->child1().useKind()) {
2152         case CellUse: {
2153             setJSValue(getById(lowCell(m_node->child1())));
2154             return;
2155         }
2156
2157         case UntypedUse: {
2158             // This is pretty weird, since we duplicate the slow path both here and in the
2159             // code generated by the IC. We should investigate making this less bad.
2160             // https://bugs.webkit.org/show_bug.cgi?id=127830
2161             LValue value = lowJSValue(m_node->child1());
2162
2163             LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped cell case"));
2164             LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped not cell case"));
2165             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetById untyped continuation"));
2166
2167             m_out.branch(
2168                 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
2169
2170             LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
2171             ValueFromBlock cellResult = m_out.anchor(getById(value));
2172             m_out.jump(continuation);
2173
2174             m_out.appendTo(notCellCase, continuation);
2175             ValueFromBlock notCellResult = m_out.anchor(vmCall(
2176                 m_out.operation(operationGetByIdGeneric),
2177                 m_callFrame, value,
2178                 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2179             m_out.jump(continuation);
2180
2181             m_out.appendTo(continuation, lastNext);
2182             setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult));
2183             return;
2184         }
2185
2186         default:
2187             DFG_CRASH(m_graph, m_node, "Bad use kind");
2188             return;
2189         }
2190     }
2191
2192     void compilePutById()
2193     {
2194         // See above; CellUse is easier so we do only that for now.
2195         ASSERT(m_node->child1().useKind() == CellUse);
2196
2197         LValue base = lowCell(m_node->child1());
2198         LValue value = lowJSValue(m_node->child2());
2199         auto uid = m_graph.identifiers()[m_node->identifierNumber()];
2200
2201         // Arguments: id, bytes, target, numArgs, args...
2202         unsigned stackmapID = m_stackmapIDs++;
2203
2204         if (verboseCompilationEnabled())
2205             dataLog("    Emitting PutById patchpoint with stackmap #", stackmapID, "\n");
2206
2207         LValue call = m_out.call(
2208             m_out.patchpointVoidIntrinsic(),
2209             m_out.constInt64(stackmapID), m_out.constInt32(sizeOfPutById()),
2210             constNull(m_out.ref8), m_out.constInt32(2), base, value);
2211         setInstructionCallingConvention(call, LLVMAnyRegCallConv);
2212
2213         m_ftlState.putByIds.append(PutByIdDescriptor(
2214             stackmapID, m_node->origin.semantic, uid,
2215             m_graph.executableFor(m_node->origin.semantic)->ecmaMode(),
2216             m_node->op() == PutByIdDirect ? Direct : NotDirect));
2217     }
2218
2219     void compileGetButterfly()
2220     {
2221         setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
2222     }
2223
2224     void compileConstantStoragePointer()
2225     {
2226         setStorage(m_out.constIntPtr(m_node->storagePointer()));
2227     }
2228
2229     void compileGetIndexedPropertyStorage()
2230     {
2231         LValue cell = lowCell(m_node->child1());
2232
2233         if (m_node->arrayMode().type() == Array::String) {
2234             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String slow case"));
2235             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String continuation"));
2236
2237             ValueFromBlock fastResult = m_out.anchor(
2238                 m_out.loadPtr(cell, m_heaps.JSString_value));
2239
2240             m_out.branch(
2241                 m_out.notNull(fastResult.value()), usually(continuation), rarely(slowPath));
2242
2243             LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
2244
2245             ValueFromBlock slowResult = m_out.anchor(
2246                 vmCall(m_out.operation(operationResolveRope), m_callFrame, cell));
2247
2248             m_out.jump(continuation);
2249
2250             m_out.appendTo(continuation, lastNext);
2251
2252             setStorage(m_out.loadPtr(m_out.phi(m_out.intPtr, fastResult, slowResult), m_heaps.StringImpl_data));
2253             return;
2254         }
2255
2256         setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
2257     }
2258
2259     void compileCheckArray()
2260     {
2261         Edge edge = m_node->child1();
2262         LValue cell = lowCell(edge);
2263
2264         if (m_node->arrayMode().alreadyChecked(m_graph, m_node, abstractValue(edge)))
2265             return;
2266
2267         speculate(
2268             BadIndexingType, jsValueValue(cell), 0,
2269             m_out.bitNot(isArrayType(cell, m_node->arrayMode())));
2270     }
2271
2272     void compileGetTypedArrayByteOffset()
2273     {
2274         LValue basePtr = lowCell(m_node->child1());
2275
2276         LBasicBlock simpleCase = FTL_NEW_BLOCK(m_out, ("wasteless typed array"));
2277         LBasicBlock wastefulCase = FTL_NEW_BLOCK(m_out, ("wasteful typed array"));
2278         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("continuation branch"));
2279
2280         LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode);
2281         m_out.branch(
2282             m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)),
2283             unsure(simpleCase), unsure(wastefulCase));
2284
2285         // begin simple case
2286         LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
2287
2288         ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
2289
2290         m_out.jump(continuation);
2291
2292         // begin wasteful case
2293         m_out.appendTo(wastefulCase, continuation);
2294
2295         LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
2296         LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly);
2297         LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
2298         LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
2299
2300         ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr));
2301
2302         m_out.jump(continuation);
2303         m_out.appendTo(continuation, lastNext);
2304
2305         // output
2306         setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut)));
2307     }
2308
2309     void compileGetArrayLength()
2310     {
2311         switch (m_node->arrayMode().type()) {
2312         case Array::Int32:
2313         case Array::Double:
2314         case Array::Contiguous: {
2315             setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
2316             return;
2317         }
2318
2319         case Array::String: {
2320             LValue string = lowCell(m_node->child1());
2321             setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length));
2322             return;
2323         }
2324
2325         case Array::DirectArguments: {
2326             LValue arguments = lowCell(m_node->child1());
2327             speculate(
2328                 ExoticObjectMode, noValue(), nullptr,
2329                 m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_overrides)));
2330             setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length));
2331             return;
2332         }
2333
2334         case Array::ScopedArguments: {
2335             LValue arguments = lowCell(m_node->child1());
2336             speculate(
2337                 ExoticObjectMode, noValue(), nullptr,
2338                 m_out.notZero8(m_out.load8(arguments, m_heaps.ScopedArguments_overrodeThings)));
2339             setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength));
2340             return;
2341         }
2342
2343         default:
2344             if (isTypedView(m_node->arrayMode().typedArrayType())) {
2345                 setInt32(
2346                     m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
2347                 return;
2348             }
2349
2350             DFG_CRASH(m_graph, m_node, "Bad array type");
2351             return;
2352         }
2353     }
2354
2355     void compileCheckInBounds()
2356     {
2357         speculate(
2358             OutOfBounds, noValue(), 0,
2359             m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2360     }
2361
2362     void compileGetByVal()
2363     {
2364         switch (m_node->arrayMode().type()) {
2365         case Array::Int32:
2366         case Array::Contiguous: {
2367             LValue index = lowInt32(m_node->child2());
2368             LValue storage = lowStorage(m_node->child3());
2369
2370             IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
2371                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
2372
2373             if (m_node->arrayMode().isInBounds()) {
2374                 LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2375                 LValue isHole = m_out.isZero64(result);
2376                 if (m_node->arrayMode().isSaneChain()) {
2377                     DFG_ASSERT(
2378                         m_graph, m_node, m_node->arrayMode().type() == Array::Contiguous);
2379                     result = m_out.select(
2380                         isHole, m_out.constInt64(JSValue::encode(jsUndefined())), result);
2381                 } else
2382                     speculate(LoadFromHole, noValue(), 0, isHole);
2383                 setJSValue(result);
2384                 return;
2385             }
2386
2387             LValue base = lowCell(m_node->child1());
2388
2389             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous fast case"));
2390             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous slow case"));
2391             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous continuation"));
2392
2393             m_out.branch(
2394                 m_out.aboveOrEqual(
2395                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2396                 rarely(slowCase), usually(fastCase));
2397
2398             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2399
2400             ValueFromBlock fastResult = m_out.anchor(
2401                 m_out.load64(baseIndex(heap, storage, index, m_node->child2())));
2402             m_out.branch(
2403                 m_out.isZero64(fastResult.value()), rarely(slowCase), usually(continuation));
2404
2405             m_out.appendTo(slowCase, continuation);
2406             ValueFromBlock slowResult = m_out.anchor(
2407                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2408             m_out.jump(continuation);
2409
2410             m_out.appendTo(continuation, lastNext);
2411             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2412             return;
2413         }
2414
2415         case Array::Double: {
2416             LValue index = lowInt32(m_node->child2());
2417             LValue storage = lowStorage(m_node->child3());
2418
2419             IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
2420
2421             if (m_node->arrayMode().isInBounds()) {
2422                 LValue result = m_out.loadDouble(
2423                     baseIndex(heap, storage, index, m_node->child2()));
2424
2425                 if (!m_node->arrayMode().isSaneChain()) {
2426                     speculate(
2427                         LoadFromHole, noValue(), 0,
2428                         m_out.doubleNotEqualOrUnordered(result, result));
2429                 }
2430                 setDouble(result);
2431                 break;
2432             }
2433
2434             LValue base = lowCell(m_node->child1());
2435
2436             LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetByVal double in bounds"));
2437             LBasicBlock boxPath = FTL_NEW_BLOCK(m_out, ("GetByVal double boxing"));
2438             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal double slow case"));
2439             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal double continuation"));
2440
2441             m_out.branch(
2442                 m_out.aboveOrEqual(
2443                     index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2444                 rarely(slowCase), usually(inBounds));
2445
2446             LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
2447             LValue doubleValue = m_out.loadDouble(
2448                 baseIndex(heap, storage, index, m_node->child2()));
2449             m_out.branch(
2450                 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
2451                 rarely(slowCase), usually(boxPath));
2452
2453             m_out.appendTo(boxPath, slowCase);
2454             ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
2455             m_out.jump(continuation);
2456
2457             m_out.appendTo(slowCase, continuation);
2458             ValueFromBlock slowResult = m_out.anchor(
2459                 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2460             m_out.jump(continuation);
2461
2462             m_out.appendTo(continuation, lastNext);
2463             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2464             return;
2465         }
2466
2467         case Array::DirectArguments: {
2468             LValue base = lowCell(m_node->child1());
2469             LValue index = lowInt32(m_node->child2());
2470
2471             speculate(
2472                 ExoticObjectMode, noValue(), nullptr,
2473                 m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_overrides)));
2474             speculate(
2475                 ExoticObjectMode, noValue(), nullptr,
2476                 m_out.aboveOrEqual(
2477                     index,
2478                     m_out.load32NonNegative(base, m_heaps.DirectArguments_length)));
2479
2480             TypedPointer address = m_out.baseIndex(
2481                 m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index));
2482             setJSValue(m_out.load64(address));
2483             return;
2484         }
2485
2486         case Array::ScopedArguments: {
2487             LValue base = lowCell(m_node->child1());
2488             LValue index = lowInt32(m_node->child2());
2489
2490             speculate(
2491                 ExoticObjectMode, noValue(), nullptr,
2492                 m_out.aboveOrEqual(
2493                     index,
2494                     m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength)));
2495
2496             LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table);
2497             LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length);
2498
2499             LBasicBlock namedCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments named case"));
2500             LBasicBlock overflowCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments overflow case"));
2501             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments continuation"));
2502
2503             m_out.branch(
2504                 m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase));
2505
2506             LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase);
2507
2508             LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope);
2509             LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments);
2510
2511             TypedPointer address = m_out.baseIndex(
2512                 m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index));
2513             LValue scopeOffset = m_out.load32(address);
2514
2515             speculate(
2516                 ExoticObjectMode, noValue(), nullptr,
2517                 m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset)));
2518
2519             address = m_out.baseIndex(
2520                 m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset));
2521             ValueFromBlock namedResult = m_out.anchor(m_out.load64(address));
2522             m_out.jump(continuation);
2523
2524             m_out.appendTo(overflowCase, continuation);
2525
2526             address = m_out.baseIndex(
2527                 m_heaps.ScopedArguments_overflowStorage, base,
2528                 m_out.zeroExtPtr(m_out.sub(index, namedLength)));
2529             LValue overflowValue = m_out.load64(address);
2530             speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue));
2531             ValueFromBlock overflowResult = m_out.anchor(overflowValue);
2532             m_out.jump(continuation);
2533
2534             m_out.appendTo(continuation, lastNext);
2535             setJSValue(m_out.phi(m_out.int64, namedResult, overflowResult));
2536             return;
2537         }
2538
2539         case Array::Generic: {
2540             setJSValue(vmCall(
2541                 m_out.operation(operationGetByVal), m_callFrame,
2542                 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
2543             return;
2544         }
2545
2546         case Array::String: {
2547             compileStringCharAt();
2548             return;
2549         }
2550
2551         default: {
2552             LValue index = lowInt32(m_node->child2());
2553             LValue storage = lowStorage(m_node->child3());
2554
2555             TypedArrayType type = m_node->arrayMode().typedArrayType();
2556
2557             if (isTypedView(type)) {
2558                 TypedPointer pointer = TypedPointer(
2559                     m_heaps.typedArrayProperties,
2560                     m_out.add(
2561                         storage,
2562                         m_out.shl(
2563                             m_out.zeroExtPtr(index),
2564                             m_out.constIntPtr(logElementSize(type)))));
2565
2566                 if (isInt(type)) {
2567                     LValue result;
2568                     switch (elementSize(type)) {
2569                     case 1:
2570                         result = m_out.load8(pointer);
2571                         break;
2572                     case 2:
2573                         result = m_out.load16(pointer);
2574                         break;
2575                     case 4:
2576                         result = m_out.load32(pointer);
2577                         break;
2578                     default:
2579                         DFG_CRASH(m_graph, m_node, "Bad element size");
2580                     }
2581
2582                     if (elementSize(type) < 4) {
2583                         if (isSigned(type))
2584                             result = m_out.signExt(result, m_out.int32);
2585                         else
2586                             result = m_out.zeroExt(result, m_out.int32);
2587                         setInt32(result);
2588                         return;
2589                     }
2590
2591                     if (isSigned(type)) {
2592                         setInt32(result);
2593                         return;
2594                     }
2595
2596                     if (m_node->shouldSpeculateInt32()) {
2597                         speculate(
2598                             Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2599                         setInt32(result);
2600                         return;
2601                     }
2602
2603                     if (m_node->shouldSpeculateMachineInt()) {
2604                         setStrictInt52(m_out.zeroExt(result, m_out.int64));
2605                         return;
2606                     }
2607
2608                     setDouble(m_out.unsignedToFP(result, m_out.doubleType));
2609                     return;
2610                 }
2611
2612                 ASSERT(isFloat(type));
2613
2614                 LValue result;
2615                 switch (type) {
2616                 case TypeFloat32:
2617                     result = m_out.fpCast(m_out.loadFloat(pointer), m_out.doubleType);
2618                     break;
2619                 case TypeFloat64:
2620                     result = m_out.loadDouble(pointer);
2621                     break;
2622                 default:
2623                     DFG_CRASH(m_graph, m_node, "Bad typed array type");
2624                 }
2625
2626                 setDouble(result);
2627                 return;
2628             }
2629
2630             DFG_CRASH(m_graph, m_node, "Bad array type");
2631             return;
2632         } }
2633     }
2634
2635     void compileGetMyArgumentByVal()
2636     {
2637         InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame;
2638
2639         LValue index = lowInt32(m_node->child2());
2640
2641         LValue limit;
2642         if (inlineCallFrame && !inlineCallFrame->isVarargs())
2643             limit = m_out.constInt32(inlineCallFrame->arguments.size() - 1);
2644         else {
2645             VirtualRegister argumentCountRegister;
2646             if (!inlineCallFrame)
2647                 argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
2648             else
2649                 argumentCountRegister = inlineCallFrame->argumentCountRegister;
2650             limit = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
2651         }
2652
2653         speculate(ExoticObjectMode, noValue(), 0, m_out.aboveOrEqual(index, limit));
2654
2655         TypedPointer base;
2656         if (inlineCallFrame) {
2657             if (inlineCallFrame->arguments.size() <= 1) {
2658                 // We should have already exited due to the bounds check, above. Just tell the
2659                 // compiler that anything dominated by this instruction is not reachable, so
2660                 // that we don't waste time generating such code. This will also plant some
2661                 // kind of crashing instruction so that if by some fluke the bounds check didn't
2662                 // work, we'll crash in an easy-to-see way.
2663                 didAlreadyTerminate();
2664                 return;
2665             }
2666             base = addressFor(inlineCallFrame->arguments[1].virtualRegister());
2667         } else
2668             base = addressFor(virtualRegisterForArgument(1));
2669
2670         LValue pointer = m_out.baseIndex(
2671             base.value(), m_out.zeroExt(index, m_out.intPtr), ScaleEight);
2672         setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)));
2673     }
2674
2675     void compilePutByVal()
2676     {
2677         Edge child1 = m_graph.varArgChild(m_node, 0);
2678         Edge child2 = m_graph.varArgChild(m_node, 1);
2679         Edge child3 = m_graph.varArgChild(m_node, 2);
2680         Edge child4 = m_graph.varArgChild(m_node, 3);
2681         Edge child5 = m_graph.varArgChild(m_node, 4);
2682
2683         switch (m_node->arrayMode().type()) {
2684         case Array::Generic: {
2685             V_JITOperation_EJJJ operation;
2686             if (m_node->op() == PutByValDirect) {
2687                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2688                     operation = operationPutByValDirectStrict;
2689                 else
2690                     operation = operationPutByValDirectNonStrict;
2691             } else {
2692                 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2693                     operation = operationPutByValStrict;
2694                 else
2695                     operation = operationPutByValNonStrict;
2696             }
2697
2698             vmCall(
2699                 m_out.operation(operation), m_callFrame,
2700                 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
2701             return;
2702         }
2703
2704         default:
2705             break;
2706         }
2707
2708         LValue base = lowCell(child1);
2709         LValue index = lowInt32(child2);
2710         LValue storage = lowStorage(child4);
2711
2712         switch (m_node->arrayMode().type()) {
2713         case Array::Int32:
2714         case Array::Double:
2715         case Array::Contiguous: {
2716             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
2717             LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
2718
2719             switch (m_node->arrayMode().type()) {
2720             case Array::Int32:
2721             case Array::Contiguous: {
2722                 LValue value = lowJSValue(child3, ManualOperandSpeculation);
2723
2724                 if (m_node->arrayMode().type() == Array::Int32)
2725                     FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
2726
2727                 TypedPointer elementPointer = m_out.baseIndex(
2728                     m_node->arrayMode().type() == Array::Int32 ?
2729                     m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
2730                     storage, m_out.zeroExtPtr(index), provenValue(child2));
2731
2732                 if (m_node->op() == PutByValAlias) {
2733                     m_out.store64(value, elementPointer);
2734                     break;
2735                 }
2736
2737                 contiguousPutByValOutOfBounds(
2738                     codeBlock()->isStrictMode()
2739                     ? operationPutByValBeyondArrayBoundsStrict
2740                     : operationPutByValBeyondArrayBoundsNonStrict,
2741                     base, storage, index, value, continuation);
2742
2743                 m_out.store64(value, elementPointer);
2744                 break;
2745             }
2746
2747             case Array::Double: {
2748                 LValue value = lowDouble(child3);
2749
2750                 FTL_TYPE_CHECK(
2751                     doubleValue(value), child3, SpecDoubleReal,
2752                     m_out.doubleNotEqualOrUnordered(value, value));
2753
2754                 TypedPointer elementPointer = m_out.baseIndex(
2755                     m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index),
2756                     provenValue(child2));
2757
2758                 if (m_node->op() == PutByValAlias) {
2759                     m_out.storeDouble(value, elementPointer);
2760                     break;
2761                 }
2762
2763                 contiguousPutByValOutOfBounds(
2764                     codeBlock()->isStrictMode()
2765                     ? operationPutDoubleByValBeyondArrayBoundsStrict
2766                     : operationPutDoubleByValBeyondArrayBoundsNonStrict,
2767                     base, storage, index, value, continuation);
2768
2769                 m_out.storeDouble(value, elementPointer);
2770                 break;
2771             }
2772
2773             default:
2774                 DFG_CRASH(m_graph, m_node, "Bad array type");
2775             }
2776
2777             m_out.jump(continuation);
2778             m_out.appendTo(continuation, outerLastNext);
2779             return;
2780         }
2781
2782         default:
2783             TypedArrayType type = m_node->arrayMode().typedArrayType();
2784
2785             if (isTypedView(type)) {
2786                 TypedPointer pointer = TypedPointer(
2787                     m_heaps.typedArrayProperties,
2788                     m_out.add(
2789                         storage,
2790                         m_out.shl(
2791                             m_out.zeroExt(index, m_out.intPtr),
2792                             m_out.constIntPtr(logElementSize(type)))));
2793
2794                 LType refType;
2795                 LValue valueToStore;
2796
2797                 if (isInt(type)) {
2798                     LValue intValue;
2799                     switch (child3.useKind()) {
2800                     case Int52RepUse:
2801                     case Int32Use: {
2802                         if (child3.useKind() == Int32Use)
2803                             intValue = lowInt32(child3);
2804                         else
2805                             intValue = m_out.castToInt32(lowStrictInt52(child3));
2806
2807                         if (isClamped(type)) {
2808                             ASSERT(elementSize(type) == 1);
2809
2810                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp atLeastZero"));
2811                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp continuation"));
2812
2813                             Vector<ValueFromBlock, 2> intValues;
2814                             intValues.append(m_out.anchor(m_out.int32Zero));
2815                             m_out.branch(
2816                                 m_out.lessThan(intValue, m_out.int32Zero),
2817                                 unsure(continuation), unsure(atLeastZero));
2818
2819                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
2820
2821                             intValues.append(m_out.anchor(m_out.select(
2822                                 m_out.greaterThan(intValue, m_out.constInt32(255)),
2823                                 m_out.constInt32(255),
2824                                 intValue)));
2825                             m_out.jump(continuation);
2826
2827                             m_out.appendTo(continuation, lastNext);
2828                             intValue = m_out.phi(m_out.int32, intValues);
2829                         }
2830                         break;
2831                     }
2832
2833                     case DoubleRepUse: {
2834                         LValue doubleValue = lowDouble(child3);
2835
2836                         if (isClamped(type)) {
2837                             ASSERT(elementSize(type) == 1);
2838
2839                             LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp atLeastZero"));
2840                             LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp withinRange"));
2841                             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp continuation"));
2842
2843                             Vector<ValueFromBlock, 3> intValues;
2844                             intValues.append(m_out.anchor(m_out.int32Zero));
2845                             m_out.branch(
2846                                 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
2847                                 unsure(continuation), unsure(atLeastZero));
2848
2849                             LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
2850                             intValues.append(m_out.anchor(m_out.constInt32(255)));
2851                             m_out.branch(
2852                                 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
2853                                 unsure(continuation), unsure(withinRange));
2854
2855                             m_out.appendTo(withinRange, continuation);
2856                             intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue)));
2857                             m_out.jump(continuation);
2858
2859                             m_out.appendTo(continuation, lastNext);
2860                             intValue = m_out.phi(m_out.int32, intValues);
2861                         } else
2862                             intValue = doubleToInt32(doubleValue);
2863                         break;
2864                     }
2865
2866                     default:
2867                         DFG_CRASH(m_graph, m_node, "Bad use kind");
2868                     }
2869
2870                     switch (elementSize(type)) {
2871                     case 1:
2872                         valueToStore = m_out.intCast(intValue, m_out.int8);
2873                         refType = m_out.ref8;
2874                         break;
2875                     case 2:
2876                         valueToStore = m_out.intCast(intValue, m_out.int16);
2877                         refType = m_out.ref16;
2878                         break;
2879                     case 4:
2880                         valueToStore = intValue;
2881                         refType = m_out.ref32;
2882                         break;
2883                     default:
2884                         DFG_CRASH(m_graph, m_node, "Bad element size");
2885                     }
2886                 } else /* !isInt(type) */ {
2887                     LValue value = lowDouble(child3);
2888                     switch (type) {
2889                     case TypeFloat32:
2890                         valueToStore = m_out.fpCast(value, m_out.floatType);
2891                         refType = m_out.refFloat;
2892                         break;
2893                     case TypeFloat64:
2894                         valueToStore = value;
2895                         refType = m_out.refDouble;
2896                         break;
2897                     default:
2898                         DFG_CRASH(m_graph, m_node, "Bad typed array type");
2899                     }
2900                 }
2901
2902                 if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias)
2903                     m_out.store(valueToStore, pointer, refType);
2904                 else {
2905                     LBasicBlock isInBounds = FTL_NEW_BLOCK(m_out, ("PutByVal typed array in bounds case"));
2906                     LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal typed array continuation"));
2907
2908                     m_out.branch(
2909                         m_out.aboveOrEqual(index, lowInt32(child5)),
2910                         unsure(continuation), unsure(isInBounds));
2911
2912                     LBasicBlock lastNext = m_out.appendTo(isInBounds, continuation);
2913                     m_out.store(valueToStore, pointer, refType);
2914                     m_out.jump(continuation);
2915
2916                     m_out.appendTo(continuation, lastNext);
2917                 }
2918
2919                 return;
2920             }
2921
2922             DFG_CRASH(m_graph, m_node, "Bad array type");
2923             break;
2924         }
2925     }
2926
2927     void compileArrayPush()
2928     {
2929         LValue base = lowCell(m_node->child1());
2930         LValue storage = lowStorage(m_node->child3());
2931
2932         switch (m_node->arrayMode().type()) {
2933         case Array::Int32:
2934         case Array::Contiguous:
2935         case Array::Double: {
2936             LValue value;
2937             LType refType;
2938
2939             if (m_node->arrayMode().type() != Array::Double) {
2940                 value = lowJSValue(m_node->child2(), ManualOperandSpeculation);
2941                 if (m_node->arrayMode().type() == Array::Int32) {
2942                     FTL_TYPE_CHECK(
2943                         jsValueValue(value), m_node->child2(), SpecInt32, isNotInt32(value));
2944                 }
2945                 refType = m_out.ref64;
2946             } else {
2947                 value = lowDouble(m_node->child2());
2948                 FTL_TYPE_CHECK(
2949                     doubleValue(value), m_node->child2(), SpecDoubleReal,
2950                     m_out.doubleNotEqualOrUnordered(value, value));
2951                 refType = m_out.refDouble;
2952             }
2953
2954             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2955
2956             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2957
2958             LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("ArrayPush fast path"));
2959             LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("ArrayPush slow path"));
2960             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPush continuation"));
2961
2962             m_out.branch(
2963                 m_out.aboveOrEqual(
2964                     prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)),
2965                 rarely(slowPath), usually(fastPath));
2966
2967             LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
2968             m_out.store(
2969                 value, m_out.baseIndex(heap, storage, m_out.zeroExtPtr(prevLength)), refType);
2970             LValue newLength = m_out.add(prevLength, m_out.int32One);
2971             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2972
2973             ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
2974             m_out.jump(continuation);
2975
2976             m_out.appendTo(slowPath, continuation);
2977             LValue operation;
2978             if (m_node->arrayMode().type() != Array::Double)
2979                 operation = m_out.operation(operationArrayPush);
2980             else
2981                 operation = m_out.operation(operationArrayPushDouble);
2982             ValueFromBlock slowResult = m_out.anchor(
2983                 vmCall(operation, m_callFrame, value, base));
2984             m_out.jump(continuation);
2985
2986             m_out.appendTo(continuation, lastNext);
2987             setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2988             return;
2989         }
2990
2991         default:
2992             DFG_CRASH(m_graph, m_node, "Bad array type");
2993             return;
2994         }
2995     }
2996
2997     void compileArrayPop()
2998     {
2999         LValue base = lowCell(m_node->child1());
3000         LValue storage = lowStorage(m_node->child2());
3001
3002         switch (m_node->arrayMode().type()) {
3003         case Array::Int32:
3004         case Array::Double:
3005         case Array::Contiguous: {
3006             IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
3007
3008             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("ArrayPop fast case"));
3009             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArrayPop slow case"));
3010             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPop continuation"));
3011
3012             LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
3013
3014             Vector<ValueFromBlock, 3> results;
3015             results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
3016             m_out.branch(
3017                 m_out.isZero32(prevLength), rarely(continuation), usually(fastCase));
3018
3019             LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
3020             LValue newLength = m_out.sub(prevLength, m_out.int32One);
3021             m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
3022             TypedPointer pointer = m_out.baseIndex(heap, storage, m_out.zeroExtPtr(newLength));
3023             if (m_node->arrayMode().type() != Array::Double) {
3024                 LValue result = m_out.load64(pointer);
3025                 m_out.store64(m_out.int64Zero, pointer);
3026                 results.append(m_out.anchor(result));
3027                 m_out.branch(
3028                     m_out.notZero64(result), usually(continuation), rarely(slowCase));
3029             } else {
3030                 LValue result = m_out.loadDouble(pointer);
3031                 m_out.store64(m_out.constInt64(bitwise_cast<int64_t>(PNaN)), pointer);
3032                 results.append(m_out.anchor(boxDouble(result)));
3033                 m_out.branch(
3034                     m_out.doubleEqual(result, result),
3035                     usually(continuation), rarely(slowCase));
3036             }
3037
3038             m_out.appendTo(slowCase, continuation);
3039             results.append(m_out.anchor(vmCall(
3040                 m_out.operation(operationArrayPopAndRecoverLength), m_callFrame, base)));
3041             m_out.jump(continuation);
3042
3043             m_out.appendTo(continuation, lastNext);
3044             setJSValue(m_out.phi(m_out.int64, results));
3045             return;
3046         }
3047
3048         default:
3049             DFG_CRASH(m_graph, m_node, "Bad array type");
3050             return;
3051         }
3052     }
3053
3054     void compileCreateActivation()
3055     {
3056         LValue scope = lowCell(m_node->child1());
3057         SymbolTable* table = m_node->castOperand<SymbolTable*>();
3058         Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure();
3059
3060         if (table->singletonScope()->isStillValid()) {
3061             LValue callResult = vmCall(
3062                 m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
3063                 scope, weakPointer(table));
3064             setJSValue(callResult);
3065             return;
3066         }
3067
3068         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateActivation slow path"));
3069         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateActivation continuation"));
3070
3071         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3072
3073         LValue fastObject = allocateObject<JSLexicalEnvironment>(
3074             JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath);
3075
3076         // We don't need memory barriers since we just fast-created the activation, so the
3077         // activation must be young.
3078         m_out.storePtr(scope, fastObject, m_heaps.JSScope_next);
3079         m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable);
3080
3081         for (unsigned i = 0; i < table->scopeSize(); ++i) {
3082             m_out.store64(
3083                 m_out.constInt64(JSValue::encode(jsUndefined())),
3084                 fastObject, m_heaps.JSEnvironmentRecord_variables[i]);
3085         }
3086
3087         ValueFromBlock fastResult = m_out.anchor(fastObject);
3088         m_out.jump(continuation);
3089
3090         m_out.appendTo(slowPath, continuation);
3091         LValue callResult = vmCall(
3092             m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
3093             scope, weakPointer(table));
3094         ValueFromBlock slowResult = m_out.anchor(callResult);
3095         m_out.jump(continuation);
3096
3097         m_out.appendTo(continuation, lastNext);
3098         setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
3099     }
3100
3101     void compileNewFunction()
3102     {
3103         LValue scope = lowCell(m_node->child1());
3104         FunctionExecutable* executable = m_node->castOperand<FunctionExecutable*>();
3105         if (executable->singletonFunction()->isStillValid()) {
3106             LValue callResult = vmCall(
3107                 m_out.operation(operationNewFunction), m_callFrame, scope, weakPointer(executable));
3108             setJSValue(callResult);
3109             return;
3110         }
3111
3112         Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->functionStructure();
3113
3114         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("NewFunction slow path"));
3115         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewFunction continuation"));
3116
3117         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3118
3119         LValue fastObject = allocateObject<JSFunction>(
3120             structure, m_out.intPtrZero, slowPath);
3121
3122         // We don't need memory barriers since we just fast-created the function, so it
3123         // must be young.
3124         m_out.storePtr(scope, fastObject, m_heaps.JSFunction_scope);
3125         m_out.storePtr(weakPointer(executable), fastObject, m_heaps.JSFunction_executable);
3126         m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.JSFunction_rareData);
3127
3128         ValueFromBlock fastResult = m_out.anchor(fastObject);
3129         m_out.jump(continuation);
3130
3131         m_out.appendTo(slowPath, continuation);
3132         LValue callResult = vmCall(
3133             m_out.operation(operationNewFunctionWithInvalidatedReallocationWatchpoint),
3134             m_callFrame, scope, weakPointer(executable));
3135         ValueFromBlock slowResult = m_out.anchor(callResult);
3136         m_out.jump(continuation);
3137
3138         m_out.appendTo(continuation, lastNext);
3139         setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
3140     }
3141
3142     void compileCreateDirectArguments()
3143     {
3144         // FIXME: A more effective way of dealing with the argument count and callee is to have
3145         // them be explicit arguments to this node.
3146         // https://bugs.webkit.org/show_bug.cgi?id=142207
3147
3148         Structure* structure =
3149             m_graph.globalObjectFor(m_node->origin.semantic)->directArgumentsStructure();
3150
3151         unsigned minCapacity = m_graph.baselineCodeBlockFor(m_node->origin.semantic)->numParameters() - 1;
3152
3153         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments slow path"));
3154         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments continuation"));
3155
3156         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3157
3158         ArgumentsLength length = getArgumentsLength();
3159
3160         LValue fastObject;
3161         if (length.isKnown) {
3162             fastObject = allocateObject<DirectArguments>(
3163                 DirectArguments::allocationSize(std::max(length.known, minCapacity)), structure,
3164                 m_out.intPtrZero, slowPath);
3165         } else {
3166             LValue size = m_out.add(
3167                 m_out.shl(length.value, m_out.constInt32(3)),
3168                 m_out.constInt32(DirectArguments::storageOffset()));
3169
3170             size = m_out.select(
3171                 m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)),
3172                 size, m_out.constInt32(DirectArguments::allocationSize(minCapacity)));
3173
3174             fastObject = allocateVariableSizedObject<DirectArguments>(
3175                 size, structure, m_out.intPtrZero, slowPath);
3176         }
3177
3178         m_out.store32(length.value, fastObject, m_heaps.DirectArguments_length);
3179         m_out.store32(m_out.constInt32(minCapacity), fastObject, m_heaps.DirectArguments_minCapacity);
3180         m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.DirectArguments_overrides);
3181
3182         ValueFromBlock fastResult = m_out.anchor(fastObject);
3183         m_out.jump(continuation);
3184
3185         m_out.appendTo(slowPath, continuation);
3186         LValue callResult = vmCall(
3187             m_out.operation(operationCreateDirectArguments), m_callFrame, weakPointer(structure),
3188             length.value, m_out.constInt32(minCapacity));
3189         ValueFromBlock slowResult = m_out.anchor(callResult);
3190         m_out.jump(continuation);
3191
3192         m_out.appendTo(continuation, lastNext);
3193         LValue result = m_out.phi(m_out.intPtr, fastResult, slowResult);
3194
3195         m_out.storePtr(getCurrentCallee(), result, m_heaps.DirectArguments_callee);
3196
3197         if (length.isKnown) {
3198             VirtualRegister start = AssemblyHelpers::argumentsStart(m_node->origin.semantic);
3199             for (unsigned i = 0; i < std::max(length.known, minCapacity); ++i) {
3200                 m_out.store64(
3201                     m_out.load64(addressFor(start + i)),
3202                     result, m_heaps.DirectArguments_storage[i]);
3203             }
3204         } else {
3205             LValue stackBase = getArgumentsStart();
3206
3207             LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments loop body"));
3208             LBasicBlock end = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments loop end"));
3209
3210             ValueFromBlock originalLength;
3211             if (minCapacity) {
3212                 LValue capacity = m_out.select(
3213                     m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)),
3214                     length.value,
3215                     m_out.constInt32(minCapacity));
3216                 originalLength = m_out.anchor(m_out.zeroExtPtr(capacity));
3217                 m_out.jump(loop);
3218             } else {
3219                 originalLength = m_out.anchor(m_out.zeroExtPtr(length.value));
3220                 m_out.branch(m_out.isNull(originalLength.value()), unsure(end), unsure(loop));
3221             }
3222
3223             lastNext = m_out.appendTo(loop, end);
3224             LValue previousIndex = m_out.phi(m_out.intPtr, originalLength);
3225             LValue index = m_out.sub(previousIndex, m_out.intPtrOne);
3226             m_out.store64(
3227                 m_out.load64(m_out.baseIndex(m_heaps.variables, stackBase, index)),
3228                 m_out.baseIndex(m_heaps.DirectArguments_storage, result, index));
3229             ValueFromBlock nextIndex = m_out.anchor(index);
3230             addIncoming(previousIndex, nextIndex);
3231             m_out.branch(m_out.isNull(index), unsure(end), unsure(loop));
3232
3233             m_out.appendTo(end, lastNext);
3234         }
3235
3236         setJSValue(result);
3237     }
3238
3239     void compileCreateScopedArguments()
3240     {
3241         LValue scope = lowCell(m_node->child1());
3242
3243         LValue result = vmCall(
3244             m_out.operation(operationCreateScopedArguments), m_callFrame,
3245             weakPointer(
3246                 m_graph.globalObjectFor(m_node->origin.semantic)->scopedArgumentsStructure()),
3247             getArgumentsStart(), getArgumentsLength().value, getCurrentCallee(), scope);
3248
3249         setJSValue(result);
3250     }
3251
3252     void compileCreateClonedArguments()
3253     {
3254         LValue result = vmCall(
3255             m_out.operation(operationCreateClonedArguments), m_callFrame,
3256             weakPointer(
3257                 m_graph.globalObjectFor(m_node->origin.semantic)->outOfBandArgumentsStructure()),
3258             getArgumentsStart(), getArgumentsLength().value, getCurrentCallee());
3259
3260         setJSValue(result);
3261     }
3262
3263     void compileNewObject()
3264     {
3265         setJSValue(allocateObject(m_node->structure()));
3266     }
3267
3268     void compileNewArray()
3269     {
3270         // First speculate appropriately on all of the children. Do this unconditionally up here
3271         // because some of the slow paths may otherwise forget to do it. It's sort of arguable
3272         // that doing the speculations up here might be unprofitable for RA - so we can consider
3273         // sinking this to below the allocation fast path if we find that this has a lot of
3274         // register pressure.
3275         for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex)
3276             speculate(m_graph.varArgChild(m_node, operandIndex));
3277
3278         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3279         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
3280             m_node->indexingType());
3281
3282         DFG_ASSERT(m_graph, m_node, structure->indexingType() == m_node->indexingType());
3283
3284         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
3285             unsigned numElements = m_node->numChildren();
3286
3287             ArrayValues arrayValues = allocateJSArray(structure, numElements);
3288
3289             for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
3290                 Edge edge = m_graph.varArgChild(m_node, operandIndex);
3291
3292                 switch (m_node->indexingType()) {
3293                 case ALL_BLANK_INDEXING_TYPES:
3294                 case ALL_UNDECIDED_INDEXING_TYPES:
3295                     DFG_CRASH(m_graph, m_node, "Bad indexing type");
3296                     break;
3297
3298                 case ALL_DOUBLE_INDEXING_TYPES:
3299                     m_out.storeDouble(
3300                         lowDouble(edge),
3301                         arrayValues.butterfly, m_heaps.indexedDoubleProperties[operandIndex]);
3302                     break;
3303
3304                 case ALL_INT32_INDEXING_TYPES:
3305                 case ALL_CONTIGUOUS_INDEXING_TYPES:
3306                     m_out.store64(
3307                         lowJSValue(edge, ManualOperandSpeculation),
3308                         arrayValues.butterfly,
3309                         m_heaps.forIndexingType(m_node->indexingType())->at(operandIndex));
3310                     break;
3311
3312                 default:
3313                     DFG_CRASH(m_graph, m_node, "Corrupt indexing type");
3314                     break;
3315                 }
3316             }
3317
3318             setJSValue(arrayValues.array);
3319             return;
3320         }
3321
3322         if (!m_node->numChildren()) {
3323             setJSValue(vmCall(
3324                 m_out.operation(operationNewEmptyArray), m_callFrame,
3325                 m_out.constIntPtr(structure)));
3326             return;
3327         }
3328
3329         size_t scratchSize = sizeof(EncodedJSValue) * m_node->numChildren();
3330         ASSERT(scratchSize);
3331         ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize);
3332         EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
3333
3334         for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
3335             Edge edge = m_graph.varArgChild(m_node, operandIndex);
3336             m_out.store64(
3337                 lowJSValue(edge, ManualOperandSpeculation),
3338                 m_out.absolute(buffer + operandIndex));
3339         }
3340
3341         m_out.storePtr(
3342             m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->activeLengthPtr()));
3343
3344         LValue result = vmCall(
3345             m_out.operation(operationNewArray), m_callFrame,
3346             m_out.constIntPtr(structure), m_out.constIntPtr(buffer),
3347             m_out.constIntPtr(m_node->numChildren()));
3348
3349         m_out.storePtr(m_out.intPtrZero, m_out.absolute(scratchBuffer->activeLengthPtr()));
3350
3351         setJSValue(result);
3352     }
3353
3354     void compileNewArrayBuffer()
3355     {
3356         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3357         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
3358             m_node->indexingType());
3359
3360         DFG_ASSERT(m_graph, m_node, structure->indexingType() == m_node->indexingType());
3361
3362         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
3363             unsigned numElements = m_node->numConstants();
3364
3365             ArrayValues arrayValues = allocateJSArray(structure, numElements);
3366
3367             JSValue* data = codeBlock()->constantBuffer(m_node->startConstant());
3368             for (unsigned index = 0; index < m_node->numConstants(); ++index) {
3369                 int64_t value;
3370                 if (hasDouble(m_node->indexingType()))
3371                     value = bitwise_cast<int64_t>(data[index].asNumber());
3372                 else
3373                     value = JSValue::encode(data[index]);
3374
3375                 m_out.store64(
3376                     m_out.constInt64(value),
3377                     arrayValues.butterfly,
3378                     m_heaps.forIndexingType(m_node->indexingType())->at(index));
3379             }
3380
3381             setJSValue(arrayValues.array);
3382             return;
3383         }
3384
3385         setJSValue(vmCall(
3386             m_out.operation(operationNewArrayBuffer), m_callFrame,
3387             m_out.constIntPtr(structure), m_out.constIntPtr(m_node->startConstant()),
3388             m_out.constIntPtr(m_node->numConstants())));
3389     }
3390
3391     void compileNewArrayWithSize()
3392     {
3393         LValue publicLength = lowInt32(m_node->child1());
3394
3395         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3396         Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
3397             m_node->indexingType());
3398
3399         if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
3400             ASSERT(
3401                 hasUndecided(structure->indexingType())
3402                 || hasInt32(structure->indexingType())
3403                 || hasDouble(structure->indexingType())
3404                 || hasContiguous(structure->indexingType()));
3405
3406             LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fast case"));
3407             LBasicBlock largeCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize large case"));
3408             LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fail case"));
3409             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize slow case"));
3410             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize continuation"));
3411
3412             m_out.branch(
3413                 m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)),
3414                 rarely(largeCase), usually(fastCase));
3415
3416             LBasicBlock lastNext = m_out.appendTo(fastCase, largeCase);
3417
3418             // We don't round up to BASE_VECTOR_LEN for new Array(blah).
3419             LValue vectorLength = publicLength;
3420
3421             LValue payloadSize =
3422                 m_out.shl(m_out.zeroExt(vectorLength, m_out.intPtr), m_out.constIntPtr(3));
3423
3424             LValue butterflySize = m_out.add(
3425                 payloadSize, m_out.constIntPtr(sizeof(IndexingHeader)));
3426
3427             LValue endOfStorage = allocateBasicStorageAndGetEnd(butterflySize, failCase);
3428
3429             LValue butterfly = m_out.sub(endOfStorage, payloadSize);
3430
3431             LValue object = allocateObject<JSArray>(
3432                 structure, butterfly, failCase);
3433
3434             m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength);
3435             m_out.store32(vectorLength, butterfly, m_heaps.Butterfly_vectorLength);
3436
3437             if (hasDouble(m_node->indexingType())) {
3438                 LBasicBlock initLoop = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init loop"));
3439                 LBasicBlock initDone = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init done"));
3440
3441                 ValueFromBlock originalIndex = m_out.anchor(vectorLength);
3442                 ValueFromBlock originalPointer = m_out.anchor(butterfly);
3443                 m_out.branch(
3444                     m_out.notZero32(vectorLength), unsure(initLoop), unsure(initDone));
3445
3446                 LBasicBlock initLastNext = m_out.appendTo(initLoop, initDone);
3447                 LValue index = m_out.phi(m_out.int32, originalIndex);
3448                 LValue pointer = m_out.phi(m_out.intPtr, originalPointer);
3449
3450                 m_out.store64(
3451                     m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
3452                     TypedPointer(m_heaps.indexedDoubleProperties.atAnyIndex(), pointer));
3453
3454                 LValue nextIndex = m_out.sub(index, m_out.int32One);
3455                 addIncoming(index, m_out.anchor(nextIndex));
3456                 addIncoming(pointer, m_out.anchor(m_out.add(pointer, m_out.intPtrEight)));
3457                 m_out.branch(
3458                     m_out.notZero32(nextIndex), unsure(initLoop), unsure(initDone));
3459
3460                 m_out.appendTo(initDone, initLastNext);
3461             }
3462
3463             ValueFromBlock fastResult = m_out.anchor(object);
3464             m_out.jump(continuation);
3465
3466             m_out.appendTo(largeCase, failCase);
3467             ValueFromBlock largeStructure = m_out.anchor(m_out.constIntPtr(
3468                 globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)));
3469             m_out.jump(slowCase);
3470
3471             m_out.appendTo(failCase, slowCase);
3472             ValueFromBlock failStructure = m_out.anchor(m_out.constIntPtr(structure));
3473             m_out.jump(slowCase);
3474
3475             m_out.appendTo(slowCase, continuation);
3476             LValue structureValue = m_out.phi(
3477                 m_out.intPtr, largeStructure, failStructure);
3478             ValueFromBlock slowResult = m_out.anchor(vmCall(
3479                 m_out.operation(operationNewArrayWithSize),
3480                 m_callFrame, structureValue, publicLength));
3481             m_out.jump(continuation);
3482
3483             m_out.appendTo(continuation, lastNext);
3484             setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
3485             return;
3486         }
3487
3488         LValue structureValue = m_out.select(
3489             m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)),
3490             m_out.constIntPtr(
3491                 globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)),
3492             m_out.constIntPtr(structure));
3493         setJSValue(vmCall(m_out.operation(operationNewArrayWithSize), m_callFrame, structureValue, publicLength));
3494     }
3495
3496     void compileAllocatePropertyStorage()
3497     {
3498         LValue object = lowCell(m_node->child1());
3499         setStorage(allocatePropertyStorage(object, m_node->transition()->previous));
3500     }
3501
3502     void compileReallocatePropertyStorage()
3503     {
3504         Transition* transition = m_node->transition();
3505         LValue object = lowCell(m_node->child1());
3506         LValue oldStorage = lowStorage(m_node->child2());
3507
3508         setStorage(
3509             reallocatePropertyStorage(
3510                 object, oldStorage, transition->previous, transition->next));
3511     }
3512
3513     void compileToStringOrCallStringConstructor()
3514     {
3515         switch (m_node->child1().useKind()) {
3516         case StringObjectUse: {
3517             LValue cell = lowCell(m_node->child1());
3518             speculateStringObjectForCell(m_node->child1(), cell);
3519             m_interpreter.filter(m_node->child1(), SpecStringObject);
3520
3521             setJSValue(m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
3522             return;
3523         }
3524
3525         case StringOrStringObjectUse: {
3526             LValue cell = lowCell(m_node->child1());
3527             LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
3528
3529             LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject not string case"));
3530             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject continuation"));
3531
3532             ValueFromBlock simpleResult = m_out.anchor(cell);
3533             m_out.branch(
3534                 m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())),
3535                 unsure(continuation), unsure(notString));
3536
3537             LBasicBlock lastNext = m_out.appendTo(notString, continuation);
3538             speculateStringObjectForStructureID(m_node->child1(), structureID);
3539             ValueFromBlock unboxedResult = m_out.anchor(
3540                 m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
3541             m_out.jump(continuation);
3542
3543             m_out.appendTo(continuation, lastNext);
3544             setJSValue(m_out.phi(m_out.int64, simpleResult, unboxedResult));
3545
3546             m_interpreter.filter(m_node->child1(), SpecString | SpecStringObject);
3547             return;
3548         }
3549
3550         case CellUse:
3551         case UntypedUse: {
3552             LValue value;
3553             if (m_node->child1().useKind() == CellUse)
3554                 value = lowCell(m_node->child1());
3555             else
3556                 value = lowJSValue(m_node->child1());
3557
3558             LBasicBlock isCell = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse is cell"));
3559             LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse not string"));
3560             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse continuation"));
3561
3562             LValue isCellPredicate;
3563             if (m_node->child1().useKind() == CellUse)
3564                 isCellPredicate = m_out.booleanTrue;
3565             else
3566                 isCellPredicate = this->isCell(value, provenType(m_node->child1()));
3567             m_out.branch(isCellPredicate, unsure(isCell), unsure(notString));
3568
3569             LBasicBlock lastNext = m_out.appendTo(isCell, notString);
3570             ValueFromBlock simpleResult = m_out.anchor(value);
3571             LValue isStringPredicate;
3572             if (m_node->child1()->prediction() & SpecString) {
3573                 isStringPredicate = isString(value, provenType(m_node->child1()));
3574             } else
3575                 isStringPredicate = m_out.booleanFalse;
3576             m_out.branch(isStringPredicate, unsure(continuation), unsure(notString));
3577
3578             m_out.appendTo(notString, continuation);
3579             LValue operation;
3580             if (m_node->child1().useKind() == CellUse)
3581                 operation = m_out.operation(m_node->op() == ToString ? operationToStringOnCell : operationCallStringConstructorOnCell);
3582             else
3583                 operation = m_out.operation(m_node->op() == ToString ? operationToString : operationCallStringConstructor);
3584             ValueFromBlock convertedResult = m_out.anchor(vmCall(operation, m_callFrame, value));
3585             m_out.jump(continuation);
3586
3587             m_out.appendTo(continuation, lastNext);
3588             setJSValue(m_out.phi(m_out.int64, simpleResult, convertedResult));
3589             return;
3590         }
3591
3592         default:
3593             DFG_CRASH(m_graph, m_node, "Bad use kind");
3594             break;
3595         }
3596     }
3597
3598     void compileToPrimitive()
3599     {
3600         LValue value = lowJSValue(m_node->child1());
3601
3602         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive cell case"));
3603         LBasicBlock isObjectCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive object case"));
3604         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToPrimitive continuation"));
3605
3606         Vector<ValueFromBlock, 3> results;
3607
3608         results.append(m_out.anchor(value));
3609         m_out.branch(
3610             isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
3611
3612         LBasicBlock lastNext = m_out.appendTo(isCellCase, isObjectCase);
3613         results.append(m_out.anchor(value));
3614         m_out.branch(
3615             isObject(value, provenType(m_node->child1())),
3616             unsure(isObjectCase), unsure(continuation));
3617
3618         m_out.appendTo(isObjectCase, continuation);
3619         results.append(m_out.anchor(vmCall(
3620             m_out.operation(operationToPrimitive), m_callFrame, value)));
3621         m_out.jump(continuation);
3622
3623         m_out.appendTo(continuation, lastNext);
3624         setJSValue(m_out.phi(m_out.int64, results));
3625     }
3626
3627     void compileMakeRope()
3628     {
3629         LValue kids[3];
3630         unsigned numKids;
3631         kids[0] = lowCell(m_node->child1());
3632         kids[1] = lowCell(m_node->child2());
3633         if (m_node->child3()) {
3634             kids[2] = lowCell(m_node->child3());
3635             numKids = 3;
3636         } else {
3637             kids[2] = 0;
3638             numKids = 2;
3639         }
3640
3641         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MakeRope slow path"));
3642         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MakeRope continuation"));
3643
3644         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3645
3646         MarkedAllocator& allocator =
3647             vm().heap.allocatorForObjectWithDestructor(sizeof(JSRopeString));
3648
3649         LValue result = allocateCell(
3650             m_out.constIntPtr(&allocator),
3651             vm().stringStructure.get(),
3652             slowPath);
3653
3654         m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSString_value);
3655         for (unsigned i = 0; i < numKids; ++i)
3656             m_out.storePtr(kids[i], result, m_heaps.JSRopeString_fibers[i]);
3657         for (unsigned i = numKids; i < JSRopeString::s_maxInternalRopeLength; ++i)
3658             m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSRopeString_fibers[i]);
3659         LValue flags = m_out.load32(kids[0], m_heaps.JSString_flags);
3660         LValue length = m_out.load32(kids[0], m_heaps.JSString_length);
3661         for (unsigned i = 1; i < numKids; ++i) {
3662             flags = m_out.bitAnd(flags, m_out.load32(kids[i], m_heaps.JSString_flags));
3663             LValue lengthAndOverflow = m_out.addWithOverflow32(
3664                 length, m_out.load32(kids[i], m_heaps.JSString_length));
3665             speculate(Uncountable, noValue(), 0, m_out.extractValue(lengthAndOverflow, 1));
3666             length = m_out.extractValue(lengthAndOverflow, 0);
3667         }
3668         m_out.store32(
3669             m_out.bitAnd(m_out.constInt32(JSString::Is8Bit), flags),
3670             result, m_heaps.JSString_flags);
3671         m_out.store32(length, result, m_heaps.JSString_length);
3672
3673         ValueFromBlock fastResult = m_out.anchor(result);
3674         m_out.jump(continuation);
3675
3676         m_out.appendTo(slowPath, continuation);
3677         ValueFromBlock slowResult;
3678         switch (numKids) {
3679         case 2:
3680             slowResult = m_out.anchor(vmCall(
3681                 m_out.operation(operationMakeRope2), m_callFrame, kids[0], kids[1]));
3682             break;
3683         case 3:
3684             slowResult = m_out.anchor(vmCall(
3685                 m_out.operation(operationMakeRope3), m_callFrame, kids[0], kids[1], kids[2]));
3686             break;
3687         default:
3688             DFG_CRASH(m_graph, m_node, "Bad number of children");
3689             break;
3690         }
3691         m_out.jump(continuation);
3692
3693         m_out.appendTo(continuation, lastNext);
3694         setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
3695     }
3696
3697     void compileStringCharAt()
3698     {
3699         LValue base = lowCell(m_node->child1());
3700         LValue index = lowInt32(m_node->child2());
3701         LValue storage = lowStorage(m_node->child3());
3702
3703         LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("GetByVal String fast path"));
3704         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetByVal String slow path"));
3705         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal String continuation"));
3706
3707         m_out.branch(
3708             m_out.aboveOrEqual(
3709                 index, m_out.load32NonNegative(base, m_heaps.JSString_length)),
3710             rarely(slowPath), usually(fastPath));
3711
3712         LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
3713
3714         LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
3715
3716         LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("GetByVal String 8-bit case"));
3717         LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("GetByVal String 16-bit case"));
3718         LBasicBlock bitsContinuation = FTL_NEW_BLOCK(m_out, ("GetByVal String bitness continuation"));
3719         LBasicBlock bigCharacter = FTL_NEW_BLOCK(m_out, ("GetByVal String big character"));
3720
3721         m_out.branch(
3722             m_out.testIsZero32(
3723                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
3724                 m_out.constInt32(StringImpl::flagIs8Bit())),
3725             unsure(is16Bit), unsure(is8Bit));
3726
3727         m_out.appendTo(is8Bit, is16Bit);
3728
3729         ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt(
3730             m_out.load8(m_out.baseIndex(
3731                 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
3732                 provenValue(m_node->child2()))),
3733             m_out.int32));
3734         m_out.jump(bitsContinuation);
3735
3736         m_out.appendTo(is16Bit, bigCharacter);
3737
3738         ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt(
3739             m_out.load16(m_out.baseIndex(
3740                 m_heaps.characters16, storage, m_out.zeroExtPtr(index),
3741                 provenValue(m_node->child2()))),
3742             m_out.int32));
3743         m_out.branch(
3744             m_out.aboveOrEqual(char16Bit.value(), m_out.constInt32(0x100)),
3745             rarely(bigCharacter), usually(bitsContinuation));
3746
3747         m_out.appendTo(bigCharacter, bitsContinuation);
3748
3749         Vector<ValueFromBlock, 4> results;
3750         results.append(m_out.anchor(vmCall(
3751             m_out.operation(operationSingleCharacterString),
3752             m_callFrame, char16Bit.value())));
3753         m_out.jump(continuation);
3754
3755         m_out.appendTo(bitsContinuation, slowPath);
3756
3757         LValue character = m_out.phi(m_out.int32, char8Bit, char16Bit);
3758
3759         LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings());
3760
3761         results.append(m_out.anchor(m_out.loadPtr(m_out.baseIndex(
3762             m_heaps.singleCharacterStrings, smallStrings, m_out.zeroExtPtr(character)))));
3763         m_out.jump(continuation);
3764
3765         m_out.appendTo(slowPath, continuation);
3766
3767         if (m_node->arrayMode().isInBounds()) {
3768             speculate(OutOfBounds, noValue(), 0, m_out.booleanTrue);
3769             results.append(m_out.anchor(m_out.intPtrZero));
3770         } else {
3771             JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3772
3773             if (globalObject->stringPrototypeChainIsSane()) {
3774                 // FIXME: This could be captured using a Speculation mode that means
3775                 // "out-of-bounds loads return a trivial value", something like
3776                 // SaneChainOutOfBounds.
3777                 // https://bugs.webkit.org/show_bug.cgi?id=144668
3778
3779                 m_graph.watchpoints().addLazily(globalObject->stringPrototype()->structure()->transitionWatchpointSet());
3780                 m_graph.watchpoints().addLazily(globalObject->objectPrototype()->structure()->transitionWatchpointSet());
3781
3782                 LBasicBlock negativeIndex = FTL_NEW_BLOCK(m_out, ("GetByVal String negative index"));
3783
3784                 results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
3785                 m_out.branch(
3786                     m_out.lessThan(index, m_out.int32Zero),
3787                     rarely(negativeIndex), usually(continuation));
3788
3789                 m_out.appendTo(negativeIndex, continuation);
3790             }
3791
3792             results.append(m_out.anchor(vmCall(
3793                 m_out.operation(operationGetByValStringInt), m_callFrame, base, index)));
3794         }
3795
3796         m_out.jump(continuation);
3797
3798         m_out.appendTo(continuation, lastNext);
3799         setJSValue(m_out.phi(m_out.int64, results));
3800     }
3801
3802     void compileStringCharCodeAt()
3803     {
3804         LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt 8-bit case"));
3805         LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt 16-bit case"));
3806         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt continuation"));
3807
3808         LValue base = lowCell(m_node->child1());
3809         LValue index = lowInt32(m_node->child2());
3810         LValue storage = lowStorage(m_node->child3());
3811
3812         speculate(
3813             Uncountable, noValue(), 0,
3814             m_out.aboveOrEqual(
3815                 index, m_out.load32NonNegative(base, m_heaps.JSString_length)));
3816
3817         LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
3818
3819         m_out.branch(
3820             m_out.testIsZero32(
3821                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
3822                 m_out.constInt32(StringImpl::flagIs8Bit())),
3823             unsure(is16Bit), unsure(is8Bit));
3824
3825         LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit);
3826
3827         ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt(
3828             m_out.load8(m_out.baseIndex(
3829                 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
3830                 provenValue(m_node->child2()))),
3831             m_out.int32));
3832         m_out.jump(continuation);
3833
3834         m_out.appendTo(is16Bit, continuation);
3835
3836         ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt(
3837             m_out.load16(m_out.baseIndex(
3838                 m_heaps.characters16, storage, m_out.zeroExtPtr(index),
3839                 provenValue(m_node->child2()))),
3840             m_out.int32));
3841         m_out.jump(continuation);
3842
3843         m_out.appendTo(continuation, lastNext);
3844
3845         setInt32(m_out.phi(m_out.int32, char8Bit, char16Bit));
3846     }
3847
3848     void compileGetByOffset()
3849     {
3850         StorageAccessData& data = m_node->storageAccessData();
3851
3852         setJSValue(loadProperty(
3853             lowStorage(m_node->child1()), data.identifierNumber, data.offset));
3854     }
3855
3856     void compileGetGetter()
3857     {
3858         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_getter));
3859     }
3860
3861     void compileGetSetter()
3862     {
3863         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_setter));
3864     }
3865
3866     void compileMultiGetByOffset()
3867     {
3868         LValue base = lowCell(m_node->child1());
3869
3870         MultiGetByOffsetData& data = m_node->multiGetByOffsetData();
3871
3872         if (data.variants.isEmpty()) {
3873             // Protect against creating a Phi function with zero inputs. LLVM doesn't like that.
3874             terminate(BadCache);
3875             return;
3876         }
3877
3878         Vector<LBasicBlock, 2> blocks(data.variants.size());
3879         for (unsigned i = data.variants.size(); i--;)
3880             blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset case ", i));
3881         LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset fail"));
3882         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset continuation"));
3883
3884         Vector<SwitchCase, 2> cases;
3885         StructureSet baseSet;
3886         for (unsigned i = data.variants.size(); i--;) {
3887             GetByIdVariant variant = data.variants[i];
3888             for (unsigned j = variant.structureSet().size(); j--;) {
3889                 Structure* structure = variant.structureSet()[j];
3890                 baseSet.add(structure);
3891                 cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1)));
3892             }
3893         }
3894         m_out.switchInstruction(
3895             m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
3896
3897         LBasicBlock lastNext = m_out.m_nextBlock;
3898
3899         Vector<ValueFromBlock, 2> results;
3900         for (unsigned i = data.variants.size(); i--;) {
3901             m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit);
3902
3903             GetByIdVariant variant = data.variants[i];
3904             baseSet.merge(variant.structureSet());
3905             LValue result;
3906             JSValue constantResult;
3907             if (variant.alternateBase()) {
3908                 constantResult = m_graph.tryGetConstantProperty(
3909                     variant.alternateBase(), variant.baseStructure(), variant.offset());
3910             }
3911             if (constantResult)
3912                 result = m_out.constInt64(JSValue::encode(constantResult));
3913             else {
3914                 LValue propertyBase;
3915                 if (variant.alternateBase())
3916                     propertyBase = weakPointer(variant.alternateBase());
3917                 else
3918                     propertyBase = base;
3919                 if (!isInlineOffset(variant.offset()))
3920                     propertyBase = m_out.loadPtr(propertyBase, m_heaps.JSObject_butterfly);
3921                 result = loadProperty(propertyBase, data.identifierNumber, variant.offset());
3922             }
3923
3924             results.append(m_out.anchor(result));
3925             m_out.jump(continuation);
3926         }
3927
3928         m_out.appendTo(exit, continuation);
3929         if (!m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet))
3930             speculate(BadCache, noValue(), nullptr, m_out.booleanTrue);
3931         m_out.unreachable();
3932
3933         m_out.appendTo(continuation, lastNext);
3934         setJSValue(m_out.phi(m_out.int64, results));
3935     }
3936
3937     void compilePutByOffset()
3938     {
3939         StorageAccessData& data = m_node->storageAccessData();
3940
3941         storeProperty(
3942             lowJSValue(m_node->child3()),
3943             lowStorage(m_node->child1()), data.identifierNumber, data.offset);
3944     }
3945
3946     void compileMultiPutByOffset()
3947     {
3948         LValue base = lowCell(m_node->child1());
3949         LValue value = lowJSValue(m_node->child2());
3950
3951         MultiPutByOffsetData& data = m_node->multiPutByOffsetData();
3952
3953         Vector<LBasicBlock, 2> blocks(data.variants.size());
3954         for (unsigned i = data.variants.size(); i--;)
3955             blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset case ", i));
3956         LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset fail"));
3957         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset continuation"));
3958
3959         Vector<SwitchCase, 2> cases;
3960         StructureSet baseSet;
3961         for (unsigned i = data.variants.size(); i--;) {
3962             PutByIdVariant variant = data.variants[i];
3963             for (unsigned j = variant.oldStructure().size(); j--;) {
3964                 Structure* structure = variant.oldStructure()[j];
3965                 baseSet.add(structure);
3966                 cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1)));
3967             }
3968         }
3969         m_out.switchInstruction(
3970             m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
3971
3972         LBasicBlock lastNext = m_out.m_nextBlock;
3973
3974         for (unsigned i = data.variants.size(); i--;) {
3975             m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit);
3976
3977             PutByIdVariant variant = data.variants[i];
3978
3979             LValue storage;
3980             if (variant.kind() == PutByIdVariant::Replace) {
3981                 if (isInlineOffset(variant.offset()))
3982                     storage = base;
3983                 else
3984                     storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly);
3985             } else {
3986                 m_graph.m_plan.transitions.addLazily(
3987                     codeBlock(), m_node->origin.semantic.codeOriginOwner(),
3988                     variant.oldStructureForTransition(), variant.newStructure());
3989
3990                 storage = storageForTransition(
3991                     base, variant.offset(),
3992                     variant.oldStructureForTransition(), variant.newStructure());
3993
3994                 ASSERT(variant.oldStructureForTransition()->indexingType() == variant.newStructure()->indexingType());
3995                 ASSERT(variant.oldStructureForTransition()->typeInfo().inlineTypeFlags() == variant.newStructure()->typeInfo().inlineTypeFlags());
3996                 ASSERT(variant.oldStructureForTransition()->typeInfo().type() == variant.newStructure()->typeInfo().type());
3997                 m_out.store32(
3998                     weakStructureID(variant.newStructure()), base, m_heaps.JSCell_structureID);
3999             }
4000
4001             storeProperty(value, storage, data.identifierNumber, variant.offset());
4002             m_out.jump(continuation);
4003         }
4004
4005         m_out.appendTo(exit, continuation);
4006         if (!m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet))
4007             speculate(BadCache, noValue(), nullptr, m_out.booleanTrue);
4008         m_out.unreachable();
4009
4010         m_out.appendTo(continuation, lastNext);
4011     }
4012
4013     void compileGetGlobalVar()
4014     {
4015         setJSValue(m_out.load64(m_out.absolute(m_node->variablePointer())));
4016     }
4017
4018     void compilePutGlobalVar()
4019     {
4020         m_out.store64(
4021             lowJSValue(m_node->child2()), m_out.absolute(m_node->variablePointer()));
4022     }
4023
4024     void compileNotifyWrite()
4025     {
4026         WatchpointSet* set = m_node->watchpointSet();
4027
4028         LBasicBlock isNotInvalidated = FTL_NEW_BLOCK(m_out, ("NotifyWrite not invalidated case"));
4029         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NotifyWrite continuation"));
4030
4031         LValue state = m_out.load8(m_out.absolute(set->addressOfState()));
4032         m_out.branch(
4033             m_out.equal(state, m_out.constInt8(IsInvalidated)),
4034             usually(continuation), rarely(isNotInvalidated));
4035
4036         LBasicBlock lastNext = m_out.appendTo(isNotInvalidated, continuation);
4037
4038         vmCall(m_out.operation(operationNotifyWrite), m_callFrame, m_out.constIntPtr(set));
4039         m_out.jump(continuation);
4040
4041         m_out.appendTo(continuation, lastNext);
4042     }
4043
4044     void compileGetCallee()
4045     {
4046         setJSValue(m_out.loadPtr(addressFor(JSStack::Callee)));
4047     }
4048
4049     void compileGetArgumentCount()
4050     {
4051         setInt32(m_out.load32(payloadFor(JSStack::ArgumentCount)));
4052     }
4053
4054     void compileGetScope()
4055     {
4056         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSFunction_scope));
4057     }
4058
4059     void compileSkipScope()
4060     {
4061         setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next));
4062     }
4063
4064     void compileGetClosureVar()
4065     {
4066         setJSValue(
4067             m_out.load64(
4068                 lowCell(m_node->child1()),
4069                 m_heaps.JSEnvironmentRecord_variables[m_node->scopeOffset().offset()]));
4070     }
4071
4072     void compilePutClosureVar()
4073     {
4074         m_out.store64(
4075             lowJSValue(m_node->child2()),
4076             lowCell(m_node->child1()),
4077             m_heaps.JSEnvironmentRecord_variables[m_node->scopeOffset().offset()]);
4078     }
4079
4080     void compileGetFromArguments()
4081     {
4082         setJSValue(
4083             m_out.load64(
4084                 lowCell(m_node->child1()),
4085                 m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()]));
4086     }
4087
4088     void compilePutToArguments()
4089     {
4090         m_out.store64(
4091             lowJSValue(m_node->child2()),
4092             lowCell(m_node->child1()),
4093             m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()]);
4094     }
4095
4096     void compileCompareEq()
4097     {
4098         if (m_node->isBinaryUseKind(Int32Use)
4099             || m_node->isBinaryUseKind(Int52RepUse)
4100             || m_node->isBinaryUseKind(DoubleRepUse)
4101             || m_node->isBinaryUseKind(ObjectUse)
4102             || m_node->isBinaryUseKind(BooleanUse)
4103             || m_node->isBinaryUseKind(StringIdentUse)) {
4104             compileCompareStrictEq();
4105             return;
4106         }
4107
4108         if (m_node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse)) {
4109             compareEqObjectOrOtherToObject(m_node->child2(), m_node->child1());
4110             return;
4111         }
4112
4113         if (m_node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse)) {
4114             compareEqObjectOrOtherToObject(m_node->child1(), m_node->child2());
4115             return;
4116         }
4117
4118         if (m_node->isBinaryUseKind(UntypedUse)) {
4119             nonSpeculativeCompare(LLVMIntEQ, operationCompareEq);
4120             return;
4121         }
4122
4123         DFG_CRASH(m_graph, m_node, "Bad use kinds");
4124     }
4125
4126     void compileCompareEqConstant()
4127     {
4128         ASSERT(m_node->child2()->asJSValue().isNull());
4129         setBoolean(
4130             equalNullOrUndefined(
4131                 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
4132     }
4133
4134     void compileCompareStrictEq()
4135     {
4136         if (m_node->isBinaryUseKind(Int32Use)) {
4137             setBoolean(
4138                 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
4139             return;
4140         }
4141
4142         if (m_node->isBinaryUseKind(Int52RepUse)) {
4143             Int52Kind kind;
4144             LValue left = lowWhicheverInt52(m_node->child1(), kind);
4145             LValue right = lowInt52(m_node->child2(), kind);
4146             setBoolean(m_out.equal(left, right));
4147             return;
4148         }
4149
4150         if (m_node->isBinaryUseKind(DoubleRepUse)) {
4151             setBoolean(
4152                 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
4153             return;
4154         }
4155
4156         if (m_node->isBinaryUseKind(StringIdentUse)) {
4157             setBoolean(
4158                 m_out.equal(lowStringIdent(m_node->child1()), lowStringIdent(m_node->child2())));
4159             return;
4160         }
4161
4162         if (m_node->isBinaryUseKind(ObjectUse, UntypedUse)) {
4163             setBoolean(
4164                 m_out.equal(
4165                     lowNonNullObject(m_node->child1()),
4166                     lowJSValue(m_node->child2())));
4167             return;
4168         }
4169
4170         if (m_node->isBinaryUseKind(UntypedUse, ObjectUse)) {
4171             setBoolean(
4172                 m_out.equal(
4173                     lowNonNullObject(m_node->child2()),
4174                     lowJSValue(m_node->child1())));
4175             return;
4176         }
4177
4178         if (m_node->isBinaryUseKind(ObjectUse)) {
4179             setBoolean(
4180                 m_out.equal(
4181                     lowNonNullObject(m_node->child1()),
4182                     lowNonNullObject(m_node->child2())));
4183             return;
4184         }
4185
4186         if (m_node->isBinaryUseKind(BooleanUse)) {
4187             setBoolean(
4188                 m_out.equal(lowBoolean(m_node->child1()), lowBoolean(m_node->child2())));
4189             return;
4190         }
4191
4192         if (m_node->isBinaryUseKind(MiscUse, UntypedUse)
4193             || m_node->isBinaryUseKind(UntypedUse, MiscUse)) {
4194             speculate(m_node->child1());
4195             speculate(m_node->child2());
4196             LValue left = lowJSValue(m_node->child1(), ManualOperandSpeculation);
4197             LValue right = lowJSValue(m_node->child2(), ManualOperandSpeculation);
4198             setBoolean(m_out.equal(left, right));
4199             return;
4200         }
4201
4202         if (m_node->isBinaryUseKind(StringIdentUse, NotStringVarUse)
4203             || m_node->isBinaryUseKind(NotStringVarUse, StringIdentUse)) {
4204             Edge leftEdge = m_node->childFor(StringIdentUse);
4205             Edge rightEdge = m_node->childFor(NotStringVarUse);
4206
4207             LValue left = lowStringIdent(leftEdge);
4208             LValue rightValue = lowJSValue(rightEdge, ManualOperandSpeculation);
4209
4210             LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is cell case"));
4211             LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is string case"));
4212             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar continuation"));
4213
4214             ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4215             m_out.branch(
4216                 isCell(rightValue, provenType(rightEdge)),
4217                 unsure(isCellCase), unsure(continuation));
4218
4219             LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
4220             ValueFromBlock notStringResult = m_out.anchor(m_out.booleanFalse);
4221             m_out.branch(
4222                 isString(rightValue, provenType(rightEdge)),
4223                 unsure(isStringCase), unsure(continuation));
4224
4225             m_out.appendTo(isStringCase, continuation);
4226             LValue right = m_out.loadPtr(rightValue, m_heaps.JSString_value);
4227             speculateStringIdent(rightEdge, rightValue, right);
4228             ValueFromBlock isStringResult = m_out.anchor(m_out.equal(left, right));
4229             m_out.jump(continuation);
4230
4231             m_out.appendTo(continuation, lastNext);
4232             setBoolean(m_out.phi(m_out.boolean, notCellResult, notStringResult, isStringResult));
4233             return;
4234         }
4235
4236         DFG_CRASH(m_graph, m_node, "Bad use kinds");
4237     }
4238
4239     void compileCompareStrictEqConstant()
4240     {
4241         JSValue constant = m_node->child2()->asJSValue();
4242
4243         setBoolean(
4244             m_out.equal(
4245                 lowJSValue(m_node->child1()),
4246                 m_out.constInt64(JSValue::encode(constant))));
4247     }
4248
4249     void compileCompareLess()
4250     {
4251         compare(LLVMIntSLT, LLVMRealOLT, operationCompareLess);
4252     }
4253
4254     void compileCompareLessEq()
4255     {
4256         compare(LLVMIntSLE, LLVMRealOLE, operationCompareLessEq);
4257     }
4258
4259     void compileCompareGreater()
4260     {
4261         compare(LLVMIntSGT, LLVMRealOGT, operationCompareGreater);
4262     }
4263
4264     void compileCompareGreaterEq()
4265     {
4266         compare(LLVMIntSGE, LLVMRealOGE, operationCompareGreaterEq);
4267     }
4268
4269     void compileLogicalNot()
4270     {
4271         setBoolean(m_out.bitNot(boolify(m_node->child1())));
4272     }
4273 #if ENABLE(FTL_NATIVE_CALL_INLINING)
4274     void compileNativeCallOrConstruct()
4275     {
4276         int numPassedArgs = m_node->numChildren() - 1;
4277         int numArgs = numPassedArgs;
4278
4279         JSFunction* knownFunction = m_node->castOperand<JSFunction*>();
4280         NativeFunction function = knownFunction->nativeFunction();
4281
4282         Dl_info info;
4283         if (!dladdr((void*)function, &info))
4284             ASSERT(false); // if we couldn't find the native function this doesn't bode well.
4285
4286         LValue callee = getFunctionBySymbol(info.dli_sname);
4287
4288         bool notInlinable;
4289         if ((notInlinable = !callee))
4290             callee = m_out.operation(function);
4291
4292         m_out.storePtr(m_callFrame, m_execStorage, m_heaps.CallFrame_callerFrame);
4293         m_out.storePtr(constNull(m_out.intPtr), addressFor(m_execStorage, JSStack::CodeBlock));
4294         m_out.storePtr(weakPointer(knownFunction), addressFor(m_execStorage, JSStack::Callee));
4295
4296         m_out.store64(m_out.constInt64(numArgs), addressFor(m_execStorage, JSStack::ArgumentCount));
4297
4298         for (int i = 0; i < numPassedArgs; ++i) {
4299             m_out.storePtr(lowJSValue(m_graph.varArgChild(m_node, 1 + i)),
4300                 addressFor(m_execStorage, JSStack::ThisArgument, i * sizeof(Register)));
4301         }
4302
4303         LValue calleeCallFrame = m_out.address(m_execState, m_heaps.CallFrame_callerFrame).value();
4304         m_out.storePtr(m_out.ptrToInt(calleeCallFrame, m_out.intPtr), m_out.absolute(&vm().topCallFrame));
4305
4306         LType typeCalleeArg;
4307         getParamTypes(getElementType(typeOf(callee)), &typeCalleeArg);
4308
4309         LValue argument = notInlinable
4310             ? m_out.ptrToInt(calleeCallFrame, typeCalleeArg)
4311             : m_out.bitCast(calleeCallFrame, typeCalleeArg);
4312         LValue call = vmCall(callee, argument);
4313
4314         if (verboseCompilationEnabled())
4315             dataLog("Native calling: ", info.dli_sname, "\n");
4316
4317         setJSValue(call);
4318     }
4319 #endif
4320
4321     void compileCallOrConstruct()
4322     {
4323         int numPassedArgs = m_node->numChildren() - 1;
4324         int numArgs = numPassedArgs;
4325
4326         LValue jsCallee = lowJSValue(m_graph.varArgChild(m_node, 0));
4327
4328         unsigned stackmapID = m_stackmapIDs++;
4329
4330         Vector<LValue> arguments;
4331         arguments.append(m_out.constInt64(stackmapID));
4332         arguments.append(m_out.constInt32(sizeOfCall()));
4333         arguments.append(constNull(m_out.ref8));
4334         arguments.append(m_out.constInt32(1 + JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + numArgs));
4335         arguments.append(jsCallee); // callee -> %rax
4336         arguments.append(getUndef(m_out.int64)); // code block
4337         arguments.append(jsCallee); // callee -> stack
4338         arguments.append(m_out.constInt64(numArgs)); // argument count and zeros for the tag
4339         for (int i = 0; i < numPassedArgs; ++i)
4340             arguments.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
4341
4342         callPreflight();
4343
4344         LValue call = m_out.call(m_out.patchpointInt64Intrinsic(), arguments);
4345         setInstructionCallingConvention(call, LLVMWebKitJSCallConv);
4346
4347         m_ftlState.jsCalls.append(JSCall(stackmapID, m_node));
4348
4349         setJSValue(call);
4350     }
4351
4352     void compileCallOrConstructVarargs()
4353     {
4354         LValue jsCallee = lowJSValue(m_node->child1());
4355         LValue thisArg = lowJSValue(m_node->child3());
4356
4357         LValue jsArguments = nullptr;
4358
4359         switch (m_node->op()) {
4360         case CallVarargs:
4361         case ConstructVarargs:
4362             jsArguments = lowJSValue(m_node->child2());
4363             break;
4364         case CallForwardVarargs:
4365         case ConstructForwardVarargs:
4366             break;
4367         default:
4368             DFG_CRASH(m_graph, m_node, "bad node type");
4369             break;
4370         }
4371
4372         unsigned stackmapID = m_stackmapIDs++;
4373
4374         Vector<LValue> arguments;
4375         arguments.append(m_out.constInt64(stackmapID));
4376         arguments.append(m_out.constInt32(sizeOfICFor(m_node)));
4377         arguments.append(constNull(m_out.ref8));
4378         arguments.append(m_out.constInt32(2 + !!jsArguments));
4379         arguments.append(jsCallee);
4380         if (jsArguments)
4381             arguments.append(jsArguments);
4382         ASSERT(thisArg);
4383         arguments.append(thisArg);
4384
4385         callPreflight();
4386
4387         LValue call = m_out.call(m_out.patchpointInt64Intrinsic(), arguments);
4388         setInstructionCallingConvention(call, LLVMCCallConv);
4389
4390         m_ftlState.jsCallVarargses.append(JSCallVarargs(stackmapID, m_node));
4391
4392         setJSValue(call);
4393     }
4394
4395     void compileLoadVarargs()
4396     {
4397         LoadVarargsData* data = m_node->loadVarargsData();
4398         LValue jsArguments = lowJSValue(m_node->child1());
4399
4400         LValue length = vmCall(
4401             m_out.operation(operationSizeOfVarargs), m_callFrame, jsArguments,
4402             m_out.constInt32(data->offset));
4403
4404         // FIXME: There is a chance that we will call an effectful length property twice. This is safe
4405         // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance
4406         // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right
4407         // past the sizing.
4408         // https://bugs.webkit.org/show_bug.cgi?id=141448
4409
4410         LValue lengthIncludingThis = m_out.add(length, m_out.int32One);
4411         speculate(
4412             VarargsOverflow, noValue(), nullptr,
4413             m_out.above(lengthIncludingThis, m_out.constInt32(data->limit)));
4414
4415         m_out.store32(lengthIncludingThis, payloadFor(data->machineCount));
4416
4417         // FIXME: This computation is rather silly. If operationLaodVarargs just took a pointer instead
4418         // of a VirtualRegister, we wouldn't have to do this.
4419         // https://bugs.webkit.org/show_bug.cgi?id=141660
4420         LValue machineStart = m_out.lShr(
4421             m_out.sub(addressFor(data->machineStart.offset()).value(), m_callFrame),
4422             m_out.constIntPtr(3));
4423
4424         vmCall(
4425             m_out.operation(operationLoadVarargs), m_callFrame,
4426             m_out.castToInt32(machineStart), jsArguments, m_out.constInt32(data->offset),
4427             length, m_out.constInt32(data->mandatoryMinimum));
4428     }
4429
4430     void compileForwardVarargs()
4431     {
4432         LoadVarargsData* data = m_node->loadVarargsData();
4433         InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame;
4434
4435         LValue length = getArgumentsLength(inlineCallFrame).value;
4436         LValue lengthIncludingThis = m_out.add(length, m_out.constInt32(1 - data->offset));
4437
4438         speculate(
4439             VarargsOverflow, noValue(), nullptr,
4440             m_out.above(lengthIncludingThis, m_out.constInt32(data->limit)));
4441
4442         m_out.store32(lengthIncludingThis, payloadFor(data->machineCount));
4443
4444         LValue sourceStart = getArgumentsStart(inlineCallFrame);
4445         LValue targetStart = addressFor(data->machineStart).value();
4446
4447         LBasicBlock undefinedLoop = FTL_NEW_BLOCK(m_out, ("ForwardVarargs undefined loop body"));
4448         LBasicBlock mainLoopEntry = FTL_NEW_BLOCK(m_out, ("ForwardVarargs main loop entry"));
4449         LBasicBlock mainLoop = FTL_NEW_BLOCK(m_out, ("ForwardVarargs main loop body"));
4450         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ForwardVarargs continuation"));
4451
4452         LValue lengthAsPtr = m_out.zeroExtPtr(length);
4453         ValueFromBlock loopBound = m_out.anchor(m_out.constIntPtr(data->mandatoryMinimum));
4454         m_out.branch(
4455             m_out.above(loopBound.value(), lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry));
4456
4457         LBasicBlock lastNext = m_out.appendTo(undefinedLoop, mainLoopEntry);
4458         LValue previousIndex = m_out.phi(m_out.intPtr, loopBound);
4459         LValue currentIndex = m_out.sub(previousIndex, m_out.intPtrOne);
4460         m_out.store64(
4461             m_out.constInt64(JSValue::encode(jsUndefined())),
4462             m_out.baseIndex(m_heaps.variables, targetStart, currentIndex));
4463         ValueFromBlock nextIndex = m_out.anchor(currentIndex);
4464         addIncoming(previousIndex, nextIndex);
4465         m_out.branch(
4466             m_out.above(currentIndex, lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry));
4467
4468         m_out.appendTo(mainLoopEntry, mainLoop);
4469         loopBound = m_out.anchor(lengthAsPtr);
4470         m_out.branch(m_out.notNull(loopBound.value()), unsure(mainLoop), unsure(continuation));
4471
4472         m_out.appendTo(mainLoop, continuation);
4473         previousIndex = m_out.phi(m_out.intPtr, loopBound);
4474         currentIndex = m_out.sub(previousIndex, m_out.intPtrOne);
4475         LValue value = m_out.load64(
4476             m_out.baseIndex(
4477                 m_heaps.variables, sourceStart,
4478                 m_out.add(currentIndex, m_out.constIntPtr(data->offset))));
4479         m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, currentIndex));
4480         nextIndex = m_out.anchor(currentIndex);
4481         addIncoming(previousIndex, nextIndex);
4482         m_out.branch(m_out.isNull(currentIndex), unsure(continuation), unsure(mainLoop));
4483
4484         m_out.appendTo(continuation, lastNext);
4485     }
4486
4487     void compileJump()
4488     {
4489         m_out.jump(lowBlock(m_node->targetBlock()));
4490     }
4491
4492     void compileBranch()
4493     {
4494         m_out.branch(
4495             boolify(m_node->child1()),
4496             WeightedTarget(
4497                 lowBlock(m_node->branchData()->taken.block),
4498                 m_node->branchData()->taken.count),
4499             WeightedTarget(
4500                 lowBlock(m_node->branchData()->notTaken.block),
4501                 m_node->branchData()->notTaken.count));
4502     }
4503
4504     void compileSwitch()
4505     {
4506         SwitchData* data = m_node->switchData();
4507         switch (data->kind) {
4508         case SwitchImm: {
4509             Vector<ValueFromBlock, 2> intValues;
4510             LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
4511
4512             LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
4513
4514             switch (m_node->child1().useKind()) {
4515             case Int32Use: {
4516                 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
4517                 m_out.jump(switchOnInts);
4518                 break;
4519             }
4520
4521             case UntypedUse: {
4522                 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
4523                 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
4524                 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
4525
4526                 LValue boxedValue = lowJSValue(m_node->child1());
4527                 m_out.branch(isNotInt32(boxedValue), unsure(isNotInt), unsure(isInt));
4528
4529                 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
4530
4531                 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
4532                 m_out.jump(switchOnInts);
4533
4534                 m_out.appendTo(isNotInt, isDouble);
4535                 m_out.branch(
4536                     isCellOrMisc(boxedValue, provenType(m_node->child1())),
4537                     usually(lowBlock(data->fallThrough.block)), rarely(isDouble));
4538
4539                 m_out.appendTo(isDouble, innerLastNext);
4540                 LValue doubleValue = unboxDouble(boxedValue);
4541                 LValue intInDouble = m_out.fpToInt32(doubleValue);
4542                 intValues.append(m_out.anchor(intInDouble));
4543                 m_out.branch(
4544                     m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
4545                     unsure(switchOnInts), unsure(lowBlock(data->fallThrough.block)));
4546                 break;
4547             }
4548
4549             default:
4550                 DFG_CRASH(m_graph, m_node, "Bad use kind");
4551                 break;
4552             }
4553
4554             m_out.appendTo(switchOnInts, lastNext);
4555             buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
4556             return;
4557         }
4558
4559         case SwitchChar: {
4560             LValue stringValue;
4561
4562             // FIXME: We should use something other than unsure() for the branch weight
4563             // of the fallThrough block. The main challenge is just that we have multiple
4564             // branches to fallThrough but a single count, so we would need to divvy it up
4565             // among the different lowered branches.
4566             // https://bugs.webkit.org/show_bug.cgi?id=129082
4567
4568             switch (m_node->child1().useKind()) {
4569             case StringUse: {
4570                 stringValue = lowString(m_node->child1());
4571                 break;
4572             }
4573
4574             case UntypedUse: {
4575                 LValue unboxedValue = lowJSValue(m_node->child1());
4576
4577                 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
4578                 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
4579
4580                 m_out.branch(
4581                     isNotCell(unboxedValue, provenType(m_node->child1())),
4582                     unsure(lowBlock(data->fallThrough.block)), unsure(isCellCase));
4583
4584                 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
4585                 LValue cellValue = unboxedValue;
4586                 m_out.branch(
4587                     isNotString(cellValue, provenType(m_node->child1())),
4588                     unsure(lowBlock(data->fallThrough.block)), unsure(isStringCase));
4589
4590                 m_out.appendTo(isStringCase, lastNext);
4591                 stringValue = cellValue;
4592                 break;
4593             }
4594
4595             default:
4596                 DFG_CRASH(m_graph, m_node, "Bad use kind");
4597                 break;
4598             }
4599
4600             LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
4601             LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
4602             LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
4603             LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
4604             LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
4605             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
4606
4607             m_out.branch(
4608                 m_out.notEqual(
4609                     m_out.load32NonNegative(stringValue, m_heaps.JSString_length),
4610                     m_out.int32One),
4611                 unsure(lowBlock(data->fallThrough.block)), unsure(lengthIs1));
4612
4613             LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
4614             Vector<ValueFromBlock, 2> values;
4615             LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
4616             values.append(m_out.anchor(fastValue));
4617             m_out.branch(m_out.isNull(fastValue), rarely(needResolution), usually(resolved));
4618
4619             m_out.appendTo(needResolution, resolved);
4620             values.append(m_out.anchor(
4621                 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
4622             m_out.jump(resolved);
4623
4624             m_out.appendTo(resolved, is8Bit);
4625             LValue value = m_out.phi(m_out.intPtr, values);
4626             LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
4627             m_out.branch(
4628                 m_out.testNonZero32(
4629                     m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
4630                     m_out.constInt32(StringImpl::flagIs8Bit())),
4631                 unsure(is8Bit), unsure(is16Bit));
4632
4633             Vector<ValueFromBlock, 2> characters;
4634             m_out.appendTo(is8Bit, is16Bit);
4635             characters.append(m_out.anchor(
4636                 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
4637             m_out.jump(continuation);
4638
4639             m_out.appendTo(is16Bit, continuation);
4640             characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
4641             m_out.jump(continuation);
4642
4643             m_out.appendTo(continuation, lastNext);
4644             buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
4645             return;
4646         }
4647
4648         case SwitchString: {
4649             switch (m_node->child1().useKind()) {
4650             case StringIdentUse: {
4651                 LValue stringImpl = lowStringIdent(m_node->child1());
4652
4653                 Vector<SwitchCase> cases;
4654                 for (unsigned i = 0; i < data->cases.size(); ++i) {
4655                     LValue value = m_out.constIntPtr(data->cases[i].value.stringImpl());
4656                     LBasicBlock block = lowBlock(data->cases[i].target.block);
4657                     Weight weight = Weight(data->cases[i].target.count);
4658                     cases.append(SwitchCase(value, block, weight));
4659                 }
4660
4661                 m_out.switchInstruction(
4662                     stringImpl, cases, lowBlock(data->fallThrough.block),
4663                     Weight(data->fallThrough.count));
4664                 return;
4665             }
4666
4667             case StringUse: {
4668                 switchString(data, lowString(m_node->child1()));
4669                 return;
4670             }
4671
4672             case UntypedUse: {
4673                 LValue value = lowJSValue(m_node->child1());
4674
4675                 LBasicBlock isCellBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString Untyped cell case"));
4676                 LBasicBlock isStringBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString Untyped string case"));
4677
4678                 m_out.branch(
4679                     isCell(value, provenType(m_node->child1())),
4680                     unsure(isCellBlock), unsure(lowBlock(data->fallThrough.block)));
4681
4682                 LBasicBlock lastNext = m_out.appendTo(isCellBlock, isStringBlock);
4683
4684                 m_out.branch(
4685                     isString(value, provenType(m_node->child1())),
4686                     unsure(isStringBlock), unsure(lowBlock(data->fallThrough.block)));
4687
4688                 m_out.appendTo(isStringBlock, lastNext);
4689
4690                 switchString(data, value);
4691                 return;
4692             }
4693
4694             default:
4695                 DFG_CRASH(m_graph, m_node, "Bad use kind");
4696                 return;
4697             }
4698             return;
4699         }
4700
4701         case SwitchCell: {
4702             LValue cell;
4703             switch (m_node->child1().useKind()) {
4704             case CellUse: {
4705                 cell = lowCell(m_node->child1());
4706                 break;
4707             }
4708
4709             case UntypedUse: {
4710                 LValue value = lowJSValue(m_node->child1());
4711                 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchCell cell case"));
4712                 m_out.branch(
4713                     isCell(value, provenType(m_node->child1())),
4714                     unsure(cellCase), unsure(lowBlock(data->fallThrough.block)));
4715                 m_out.appendTo(cellCase);
4716                 cell = value;
4717                 break;
4718             }
4719
4720             default:
4721                 DFG_CRASH(m_graph, m_node, "Bad use kind");
4722                 return;
4723             }
4724
4725             buildSwitch(m_node->switchData(), m_out.intPtr, cell);
4726             return;
4727         } }
4728
4729         DFG_CRASH(m_graph, m_node, "Bad switch kind");
4730     }
4731
4732     void compileReturn()
4733     {
4734         m_out.ret(lowJSValue(m_node->child1()));
4735     }
4736
4737     void compileForceOSRExit()
4738     {
4739         terminate(InadequateCoverage);
4740     }
4741
4742     void compileThrow()
4743     {
4744         terminate(Uncountable);
4745     }
4746
4747     void compileInvalidationPoint()
4748     {
4749         if (verboseCompilationEnabled())
4750             dataLog("    Invalidation point with availability: ", availabilityMap(), "\n");
4751
4752         m_ftlState.jitCode->osrExit.append(OSRExit(
4753             UncountableInvalidation, InvalidValueFormat, MethodOfGettingAValueProfile(),
4754             m_codeOriginForExitTarget, m_codeOriginForExitProfile,
4755             availabilityMap().m_locals.numberOfArguments(),
4756             availabilityMap().m_locals.numberOfLocals()));
4757         m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo());
4758
4759         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
4760         OSRExitCompilationInfo& info = m_ftlState.finalizer->osrExit.last();
4761
4762         ExitArgumentList arguments;
4763
4764         buildExitArguments(exit, arguments, FormattedValue(), exit.m_codeOrigin);
4765         callStackmap(exit, arguments);
4766
4767         info.m_isInvalidationPoint = true;
4768     }
4769
4770     void compileIsUndefined()
4771     {
4772         setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
4773     }
4774
4775     void compileIsBoolean()
4776     {
4777         setBoolean(isBoolean(lowJSValue(m_node->child1()), provenType(m_node->child1())));
4778     }
4779
4780     void compileIsNumber()
4781     {
4782         setBoolean(isNumber(lowJSValue(m_node->child1()), provenType(m_node->child1())));
4783     }
4784
4785     void compileIsString()
4786     {
4787         LValue value = lowJSValue(m_node->child1());
4788
4789         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("IsString cell case"));
4790         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsString continuation"));
4791
4792         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4793         m_out.branch(
4794             isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
4795
4796         LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
4797         ValueFromBlock cellResult = m_out.anchor(isString(value, provenType(m_node->child1())));
4798         m_out.jump(continuation);
4799
4800         m_out.appendTo(continuation, lastNext);
4801         setBoolean(m_out.phi(m_out.boolean, notCellResult, cellResult));
4802     }
4803
4804     void compileIsObject()
4805     {
4806         LValue value = lowJSValue(m_node->child1());
4807
4808         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("IsObject cell case"));
4809         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsObject continuation"));
4810
4811         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4812         m_out.branch(
4813             isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
4814
4815         LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
4816         ValueFromBlock cellResult = m_out.anchor(isObject(value, provenType(m_node->child1())));
4817         m_out.jump(continuation);
4818
4819         m_out.appendTo(continuation, lastNext);
4820         setBoolean(m_out.phi(m_out.boolean, notCellResult, cellResult));
4821     }
4822
4823     void compileIsObjectOrNull()
4824     {
4825         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
4826
4827         Edge child = m_node->child1();
4828         LValue value = lowJSValue(child);
4829
4830         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull cell case"));
4831         LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull not function case"));
4832         LBasicBlock objectCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull object case"));
4833         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull slow path"));
4834         LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull not cell case"));
4835         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull continuation"));
4836
4837         m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase));
4838
4839         LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase);
4840         ValueFromBlock isFunctionResult = m_out.anchor(m_out.booleanFalse);
4841         m_out.branch(
4842             isFunction(value, provenType(child)),
4843             unsure(continuation), unsure(notFunctionCase));
4844
4845         m_out.appendTo(notFunctionCase, objectCase);
4846         ValueFromBlock notObjectResult = m_out.anchor(m_out.booleanFalse);
4847         m_out.branch(
4848             isObject(value, provenType(child)),
4849             unsure(objectCase), unsure(continuation));
4850
4851         m_out.appendTo(objectCase, slowPath);
4852         ValueFromBlock objectResult = m_out.anchor(m_out.booleanTrue);
4853         m_out.branch(
4854             isExoticForTypeof(value, provenType(child)),
4855             rarely(slowPath), usually(continuation));
4856
4857         m_out.appendTo(slowPath, notCellCase);
4858         LValue slowResultValue = vmCall(
4859             m_out.operation(operationObjectIsObject), m_callFrame, weakPointer(globalObject),
4860             value);
4861         ValueFromBlock slowResult = m_out.anchor(m_out.notNull(slowResultValue));
4862         m_out.jump(continuation);
4863
4864         m_out.appendTo(notCellCase, continuation);
4865         LValue notCellResultValue = m_out.equal(value, m_out.constInt64(JSValue::encode(jsNull())));
4866         ValueFromBlock notCellResult = m_out.anchor(notCellResultValue);
4867         m_out.jump(continuation);
4868
4869         m_out.appendTo(continuation, lastNext);
4870         LValue result = m_out.phi(
4871             m_out.boolean,
4872             isFunctionResult, notObjectResult, objectResult, slowResult, notCellResult);
4873         setBoolean(result);
4874     }
4875
4876     void compileIsFunction()
4877     {
4878         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
4879
4880         Edge child = m_node->child1();
4881         LValue value = lowJSValue(child);
4882
4883         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("IsFunction cell case"));
4884         LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("IsFunction not function case"));
4885         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("IsFunction slow path"));
4886         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsFunction continuation"));
4887
4888         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4889         m_out.branch(
4890             isCell(value, provenType(child)), unsure(cellCase), unsure(continuation));
4891
4892         LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase);
4893         ValueFromBlock functionResult = m_out.anchor(m_out.booleanTrue);
4894         m_out.branch(
4895             isFunction(value, provenType(child)),
4896             unsure(continuation), unsure(notFunctionCase));
4897
4898         m_out.appendTo(notFunctionCase, slowPath);
4899         ValueFromBlock objectResult = m_out.anchor(m_out.booleanFalse);
4900         m_out.branch(
4901             isExoticForTypeof(value, provenType(child)),
4902             rarely(slowPath), usually(continuation));
4903
4904         m_out.appendTo(slowPath, continuation);
4905         LValue slowResultValue = vmCall(
4906             m_out.operation(operationObjectIsFunction), m_callFrame, weakPointer(globalObject),
4907             value);
4908         ValueFromBlock slowResult = m_out.anchor(m_out.notNull(slowResultValue));
4909         m_out.jump(continuation);
4910
4911         m_out.appendTo(continuation, lastNext);
4912         LValue result = m_out.phi(
4913             m_out.boolean, notCellResult, functionResult, objectResult, slowResult);
4914         setBoolean(result);
4915     }
4916
4917     void compileTypeOf()
4918     {
4919         Edge child = m_node->child1();
4920         LValue value = lowJSValue(child);
4921
4922         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("TypeOf continuation"));
4923         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
4924
4925         Vector<ValueFromBlock> results;
4926
4927         buildTypeOf(
4928             child, value,
4929             [&] (TypeofType type) {
4930                 results.append(m_out.anchor(weakPointer(vm().smallStrings.typeString(type))));
4931                 m_out.jump(continuation);
4932             });
4933
4934         m_out.appendTo(continuation, lastNext);
4935         setJSValue(m_out.phi(m_out.int64, results));
4936     }
4937
4938     void compileIn()
4939     {
4940         Edge base = m_node->child2();
4941         LValue cell = lowCell(base);
4942         speculateObject(base, cell);
4943         if (JSString* string = m_node->child1()->dynamicCastConstant<JSString*>()) {
4944             if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) {
4945
4946                 const auto str = static_cast<const AtomicStringImpl*>(string->tryGetValueImpl());
4947                 unsigned stackmapID = m_stackmapIDs++;
4948
4949                 LValue call = m_out.call(
4950                     m_out.patchpointInt64Intrinsic(),
4951                     m_out.constInt64(stackmapID), m_out.constInt32(sizeOfIn()),
4952                     constNull(m_out.ref8), m_out.constInt32(1), cell);
4953
4954                 setInstructionCallingConvention(call, LLVMAnyRegCallConv);
4955
4956                 m_ftlState.checkIns.append(CheckInDescriptor(stackmapID, m_node->origin.semantic, str));
4957                 setJSValue(call);
4958                 return;
4959             }
4960         }
4961
4962         setJSValue(vmCall(m_out.operation(operationGenericIn), m_callFrame, cell, lowJSValue(m_node->child1())));
4963     }
4964
4965     void compileCheckHasInstance()
4966     {
4967         speculate(
4968             Uncountable, noValue(), 0,
4969             m_out.testIsZero8(
4970                 m_out.load8(lowCell(m_node->child1()), m_heaps.JSCell_typeInfoFlags),
4971                 m_out.constInt8(ImplementsDefaultHasInstance)));
4972     }
4973
4974     void compileInstanceOf()
4975     {
4976         LValue cell;
4977
4978         if (m_node->child1().useKind() == UntypedUse)
4979             cell = lowJSValue(m_node->child1());
4980         else
4981             cell = lowCell(m_node->child1());
4982
4983         LValue prototype = lowCell(m_node->child2());
4984
4985         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("InstanceOf cell case"));
4986         LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("InstanceOf loop"));
4987         LBasicBlock notYetInstance = FTL_NEW_BLOCK(m_out, ("InstanceOf not yet instance"));
4988         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("InstanceOf continuation"));
4989
4990         LValue condition;
4991         if (m_node->child1().useKind() == UntypedUse)
4992             condition = isCell(cell, provenType(m_node->child1()));
4993         else
4994             condition = m_out.booleanTrue;
4995
4996         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4997         m_out.branch(condition, unsure(isCellCase), unsure(continuation));
4998
4999         LBasicBlock lastNext = m_out.appendTo(isCellCase, loop);
5000
5001         speculate(BadType, noValue(), 0, isNotObject(prototype, provenType(m_node->child2())));
5002
5003         ValueFromBlock originalValue = m_out.anchor(cell);
5004         m_out.jump(loop);
5005
5006         m_out.appendTo(loop, notYetInstance);
5007         LValue value = m_out.phi(m_out.int64, originalValue);
5008         LValue structure = loadStructure(value);
5009         LValue currentPrototype = m_out.load64(structure, m_heaps.Structure_prototype);
5010         ValueFromBlock isInstanceResult = m_out.anchor(m_out.booleanTrue);
5011         m_out.branch(
5012             m_out.equal(currentPrototype, prototype),
5013             unsure(continuation), unsure(notYetInstance));
5014
5015         m_out.appendTo(notYetInstance, continuation);
5016         ValueFromBlock notInstanceResult = m_out.anchor(m_out.booleanFalse);
5017         addIncoming(value, m_out.anchor(currentPrototype));
5018         m_out.branch(isCell(currentPrototype), unsure(loop), unsure(continuation));
5019
5020         m_out.appendTo(continuation, lastNext);
5021         setBoolean(
5022             m_out.phi(m_out.boolean, notCellResult, isInstanceResult, notInstanceResult));
5023     }
5024
5025     void compileCountExecution()
5026     {
5027         TypedPointer counter = m_out.absolute(m_node->executionCounter()->address());
5028         m_out.store64(m_out.add(m_out.load64(counter), m_out.constInt64(1)), counter);
5029     }
5030
5031     void compileStoreBarrier()
5032     {
5033         emitStoreBarrier(lowCell(m_node->child1()));
5034     }
5035
5036     void compileHasIndexedProperty()
5037     {
5038         switch (m_node->arrayMode().type()) {
5039         case Array::Int32:
5040         case Array::Contiguous: {
5041             LValue base = lowCell(m_node->child1());
5042             LValue index = lowInt32(m_node->child2());
5043             LValue storage = lowStorage(m_node->child3());
5044
5045             IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
5046                 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
5047
5048             LBasicBlock checkHole = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous check hole"));
5049             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous slow case"));
5050             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous continuation"));
5051
5052             if (!m_node->arrayMode().isInBounds()) {
5053                 m_out.branch(
5054                     m_out.aboveOrEqual(
5055                         index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
5056                     rarely(slowCase), usually(checkHole));
5057             } else
5058                 m_out.jump(checkHole);
5059
5060             LBasicBlock lastNext = m_out.appendTo(checkHole, slowCase);
5061             ValueFromBlock checkHoleResult = m_out.anchor(
5062                 m_out.notZero64(m_out.load64(baseIndex(heap, storage, index, m_node->child2()))));
5063             m_out.branch(checkHoleResult.value(), usually(continuation), rarely(slowCase));
5064
5065             m_out.appendTo(slowCase, continuation);
5066             ValueFromBlock slowResult = m_out.anchor(m_out.equal(
5067                 m_out.constInt64(JSValue::encode(jsBoolean(true))),
5068                 vmCall(m_out.operation(operationHasIndexedProperty), m_callFrame, base, index)));
5069             m_out.jump(continuation);
5070
5071             m_out.appendTo(continuation, lastNext);
5072             setBoolean(m_out.phi(m_out.boolean, checkHoleResult, slowResult));
5073             return;
5074         }
5075         case Array::Double: {
5076             LValue base = lowCell(m_node->child1());
5077             LValue index = lowInt32(m_node->child2());
5078             LValue storage = lowStorage(m_node->child3());
5079
5080             IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
5081
5082             LBasicBlock checkHole = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double check hole"));
5083             LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double slow case"));
5084             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double continuation"));
5085
5086             if (!m_node->arrayMode().isInBounds()) {
5087                 m_out.branch(
5088                     m_out.aboveOrEqual(
5089                         index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
5090                     rarely(slowCase), usually(checkHole));
5091             } else
5092                 m_out.jump(checkHole);
5093
5094             LBasicBlock lastNext = m_out.appendTo(checkHole, slowCase);
5095             LValue doubleValue = m_out.loadDouble(baseIndex(heap, storage, index, m_node->child2()));
5096             ValueFromBlock checkHoleResult = m_out.anchor(m_out.doubleEqual(doubleValue, doubleValue));
5097             m_out.branch(checkHoleResult.value(), usually(continuation), rarely(slowCase));
5098
5099             m_out.appendTo(slowCase, continuation);
5100             ValueFromBlock slowResult = m_out.anchor(m_out.equal(
5101                 m_out.constInt64(JSValue::encode(jsBoolean(true))),
5102                 vmCall(m_out.operation(operationHasIndexedProperty), m_callFrame, base, index)));
5103             m_out.jump(continuation);
5104
5105             m_out.appendTo(continuation, lastNext);
5106             setBoolean(m_out.phi(m_out.boolean, checkHoleResult, slowResult));
5107             return;
5108         }
5109
5110         default:
5111             RELEASE_ASSERT_NOT_REACHED();
5112             return;
5113         }
5114     }
5115
5116     void compileHasGenericProperty()
5117     {
5118         LValue base = lowJSValue(m_node->child1());
5119         LValue property = lowCell(m_node->child2());
5120         setJSValue(vmCall(m_out.operation(operationHasGenericProperty), m_callFrame, base, property));
5121     }
5122
5123     void compileHasStructureProperty()
5124     {
5125         LValue base = lowJSValue(m_node->child1());
5126         LValue property = lowString(m_node->child2());
5127         LValue enumerator = lowCell(m_node->child3());
5128
5129         LBasicBlock correctStructure = FTL_NEW_BLOCK(m_out, ("HasStructureProperty correct structure"));
5130         LBasicBlock wrongStructure = FTL_NEW_BLOCK(m_out, ("HasStructureProperty wrong structure"));
5131         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasStructureProperty continuation"));
5132
5133         m_out.branch(m_out.notEqual(
5134             m_out.load32(base, m_heaps.JSCell_structureID),
5135             m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)),
5136             rarely(wrongStructure), usually(correctStructure));
5137
5138         LBasicBlock lastNext = m_out.appendTo(correctStructure, wrongStructure);
5139         ValueFromBlock correctStructureResult = m_out.anchor(m_out.booleanTrue);
5140         m_out.jump(continuation);
5141
5142         m_out.appendTo(wrongStructure, continuation);
5143         ValueFromBlock wrongStructureResult = m_out.anchor(
5144             m_out.equal(
5145                 m_out.constInt64(JSValue::encode(jsBoolean(true))),
5146                 vmCall(m_out.operation(operationHasGenericProperty), m_callFrame, base, property)));
5147         m_out.jump(continuation);
5148
5149         m_out.appendTo(continuation, lastNext);
5150         setBoolean(m_out.phi(m_out.boolean, correctStructureResult, wrongStructureResult));
5151     }
5152
5153     void compileGetDirectPname()
5154     {
5155         LValue base = lowCell(m_graph.varArgChild(m_node, 0));
5156         LValue property = lowCell(m_graph.varArgChild(m_node, 1));
5157         LValue index = lowInt32(m_graph.varArgChild(m_node, 2));
5158         LValue enumerator = lowCell(m_graph.varArgChild(m_node, 3));
5159
5160         LBasicBlock checkOffset = FTL_NEW_BLOCK(m_out, ("GetDirectPname check offset"));
5161         LBasicBlock inlineLoad = FTL_NEW_BLOCK(m_out, ("GetDirectPname inline load"));
5162         LBasicBlock outOfLineLoad = FTL_NEW_BLOCK(m_out, ("GetDirectPname out-of-line load"));
5163         LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetDirectPname slow case"));
5164         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetDirectPname continuation"));
5165
5166         m_out.branch(m_out.notEqual(
5167             m_out.load32(base, m_heaps.JSCell_structureID),
5168             m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)),
5169             rarely(slowCase), usually(checkOffset));
5170
5171         LBasicBlock lastNext = m_out.appendTo(checkOffset, inlineLoad);
5172         m_out.branch(m_out.aboveOrEqual(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity)),
5173             unsure(outOfLineLoad), unsure(inlineLoad));
5174
5175         m_out.appendTo(inlineLoad, outOfLineLoad);
5176         ValueFromBlock inlineResult = m_out.anchor(
5177             m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(),
5178                 base, m_out.zeroExt(index, m_out.int64), ScaleEight, JSObject::offsetOfInlineStorage())));
5179         m_out.jump(continuation);
5180
5181         m_out.appendTo(outOfLineLoad, slowCase);
5182         LValue storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly);
5183         LValue realIndex = m_out.signExt(
5184             m_out.neg(m_out.sub(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity))),
5185             m_out.int64);
5186         int32_t offsetOfFirstProperty = static_cast<int32_t>(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue);
5187         ValueFromBlock outOfLineResult = m_out.anchor(
5188             m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), storage, realIndex, ScaleEight, offsetOfFirstProperty)));
5189         m_out.jump(continuation);
5190
5191         m_out.appendTo(slowCase, continuation);
5192         ValueFromBlock slowCaseResult = m_out.anchor(
5193             vmCall(m_out.operation(operationGetByVal), m_callFrame, base, property));
5194         m_out.jump(continuation);
5195
5196         m_out.appendTo(continuation, lastNext);
5197         setJSValue(m_out.phi(m_out.int64, inlineResult, outOfLineResult, slowCaseResult));
5198     }
5199
5200     void compileGetEnumerableLength()
5201     {
5202         LValue enumerator = lowCell(m_node->child1());
5203         setInt32(m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_indexLength));
5204     }
5205
5206     void compileGetPropertyEnumerator()
5207     {
5208         LValue base = lowCell(m_node->child1());
5209         setJSValue(vmCall(m_out.operation(operationGetPropertyEnumerator), m_callFrame, base));
5210     }
5211
5212     void compileGetEnumeratorStructurePname()
5213     {
5214         LValue enumerator = lowCell(m_node->child1());
5215         LValue index = lowInt32(m_node->child2());
5216
5217         LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname in bounds"));
5218         LBasicBlock outOfBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname out of bounds"));
5219         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname continuation"));
5220
5221         m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endStructurePropertyIndex)),
5222             usually(inBounds), rarely(outOfBounds));
5223
5224         LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds);
5225         LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector);
5226         ValueFromBlock inBoundsResult = m_out.anchor(
5227             m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index))));
5228         m_out.jump(continuation);
5229
5230         m_out.appendTo(outOfBounds, continuation);
5231         ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueNull));
5232         m_out.jump(continuation);
5233
5234         m_out.appendTo(continuation, lastNext);
5235         setJSValue(m_out.phi(m_out.int64, inBoundsResult, outOfBoundsResult));
5236     }
5237
5238     void compileGetEnumeratorGenericPname()
5239     {
5240         LValue enumerator = lowCell(m_node->child1());
5241         LValue index = lowInt32(m_node->child2());
5242
5243         LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname in bounds"));
5244         LBasicBlock outOfBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname out of bounds"));
5245         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname continuation"));
5246
5247         m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endGenericPropertyIndex)),
5248             usually(inBounds), rarely(outOfBounds));
5249
5250         LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds);
5251         LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector);
5252         ValueFromBlock inBoundsResult = m_out.anchor(
5253             m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index))));
5254         m_out.jump(continuation);
5255
5256         m_out.appendTo(outOfBounds, continuation);
5257         ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueNull));
5258         m_out.jump(continuation);
5259
5260         m_out.appendTo(continuation, lastNext);
5261         setJSValue(m_out.phi(m_out.int64, inBoundsResult, outOfBoundsResult));
5262     }
5263
5264     void compileToIndexString()
5265     {
5266         LValue index = lowInt32(m_node->child1());
5267         setJSValue(vmCall(m_out.operation(operationToIndexString), m_callFrame, index));
5268     }
5269
5270     void compileCheckStructureImmediate()
5271     {
5272         LValue structure = lowCell(m_node->child1());
5273         checkStructure(
5274             structure, noValue(), BadCache, m_node->structureSet(),
5275             [this] (Structure* structure) {
5276                 return weakStructure(structure);
5277             });
5278     }
5279
5280     void compileMaterializeNewObject()
5281     {
5282         ObjectMaterializationData& data = m_node->objectMaterializationData();
5283
5284         // Lower the values first, to avoid creating values inside a control flow diamond.
5285
5286         Vector<LValue, 8> values;
5287         for (unsigned i = 0; i < data.m_properties.size(); ++i)
5288             values.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
5289
5290         StructureSet set;
5291         m_interpreter.phiChildren()->forAllTransitiveIncomingValues(
5292             m_graph.varArgChild(m_node, 0).node(),
5293             [&] (Node* incoming) {
5294                 set.add(incoming->castConstant<Structure*>());
5295             });
5296
5297         Vector<LBasicBlock, 1> blocks(set.size());
5298         for (unsigned i = set.size(); i--;)
5299             blocks[i] = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject case ", i));
5300         LBasicBlock dummyDefault = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject default case"));
5301         LBasicBlock outerContinuation = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject continuation"));
5302
5303         Vector<SwitchCase, 1> cases(set.size());
5304         for (unsigned i = set.size(); i--;)
5305             cases[i] = SwitchCase(weakStructure(set[i]), blocks[i], Weight(1));
5306         m_out.switchInstruction(
5307             lowCell(m_graph.varArgChild(m_node, 0)), cases, dummyDefault, Weight(0));
5308
5309         LBasicBlock outerLastNext = m_out.m_nextBlock;
5310
5311         Vector<ValueFromBlock, 1> results;
5312
5313         for (unsigned i = set.size(); i--;) {
5314             m_out.appendTo(blocks[i], i + 1 < set.size() ? blocks[i + 1] : dummyDefault);
5315
5316             Structure* structure = set[i];
5317
5318             LValue object;
5319             LValue butterfly;
5320
5321             if (structure->outOfLineCapacity()) {
5322                 size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
5323                 MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize);
5324
5325                 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject complex object allocation slow path"));
5326                 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject complex object allocation continuation"));
5327
5328                 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5329
5330                 LValue endOfStorage = allocateBasicStorageAndGetEnd(
5331                     m_out.constIntPtr(structure->outOfLineCapacity() * sizeof(JSValue)),
5332                     slowPath);
5333
5334                 LValue fastButterflyValue = m_out.add(
5335                     m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage);
5336
5337                 LValue fastObjectValue = allocateObject(
5338                     m_out.constIntPtr(allocator), structure, fastButterflyValue, slowPath);
5339
5340                 ValueFromBlock fastObject = m_out.anchor(fastObjectValue);
5341                 ValueFromBlock fastButterfly = m_out.anchor(fastButterflyValue);
5342                 m_out.jump(continuation);
5343
5344                 m_out.appendTo(slowPath, continuation);
5345
5346                 ValueFromBlock slowObject = m_out.anchor(vmCall(
5347                     m_out.operation(operationNewObjectWithButterfly),
5348                     m_callFrame, m_out.constIntPtr(structure)));
5349                 ValueFromBlock slowButterfly = m_out.anchor(
5350                     m_out.loadPtr(slowObject.value(), m_heaps.JSObject_butterfly));
5351
5352                 m_out.jump(continuation);
5353
5354                 m_out.appendTo(continuation, lastNext);
5355
5356                 object = m_out.phi(m_out.intPtr, fastObject, slowObject);
5357                 butterfly = m_out.phi(m_out.intPtr, fastButterfly, slowButterfly);
5358             } else {
5359                 // In the easy case where we can do a one-shot allocation, we simply allocate the
5360                 // object to directly have the desired structure.
5361                 object = allocateObject(structure);
5362                 butterfly = nullptr; // Don't have one, don't need one.
5363             }
5364
5365             for (PropertyMapEntry entry : structure->getPropertiesConcurrently()) {
5366                 for (unsigned i = data.m_properties.size(); i--;) {
5367                     PhantomPropertyValue value = data.m_properties[i];
5368                     if (m_graph.identifiers()[value.m_identifierNumber] != entry.key)
5369                         continue;
5370
5371                     LValue base = isInlineOffset(entry.offset) ? object : butterfly;
5372                     storeProperty(values[i], base, value.m_identifierNumber, entry.offset);
5373                     break;
5374                 }
5375             }
5376
5377             results.append(m_out.anchor(object));
5378             m_out.jump(outerContinuation);
5379         }
5380
5381         m_out.appendTo(dummyDefault, outerContinuation);
5382         m_out.unreachable();
5383
5384         m_out.appendTo(outerContinuation, outerLastNext);
5385         setJSValue(m_out.phi(m_out.intPtr, results));
5386     }
5387
5388     void compileMaterializeCreateActivation()
5389     {
5390         ObjectMaterializationData& data = m_node->objectMaterializationData();
5391
5392         Vector<LValue, 8> values;
5393         for (unsigned i = 0; i < data.m_properties.size(); ++i)
5394             values.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
5395
5396         LValue scope = lowCell(m_graph.varArgChild(m_node, 0));
5397         SymbolTable* table = m_node->castOperand<SymbolTable*>();
5398         Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure();
5399
5400         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MaterializeCreateActivation slow path"));
5401         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MaterializeCreateActivation continuation"));
5402
5403         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5404
5405         LValue fastObject = allocateObject<JSLexicalEnvironment>(
5406             JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath);
5407
5408         m_out.storePtr(scope, fastObject, m_heaps.JSScope_next);
5409         m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable);
5410
5411
5412         ValueFromBlock fastResult = m_out.anchor(fastObject);
5413         m_out.jump(continuation);
5414
5415         m_out.appendTo(slowPath, continuation);
5416         LValue callResult = vmCall(
5417             m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
5418             scope, weakPointer(table));
5419         ValueFromBlock slowResult =  m_out.anchor(callResult);
5420         m_out.jump(continuation);
5421
5422         m_out.appendTo(continuation, lastNext);
5423         LValue activation = m_out.phi(m_out.intPtr, fastResult, slowResult);
5424         RELEASE_ASSERT(data.m_properties.size() == table->scopeSize());
5425         for (unsigned i = 0; i < data.m_properties.size(); ++i) {
5426             m_out.store64(values[i],
5427                 activation,
5428                 m_heaps.JSEnvironmentRecord_variables[data.m_properties[i].m_identifierNumber]);
5429         }
5430
5431         if (validationEnabled()) {
5432             // Validate to make sure every slot in the scope has one value.
5433             ConcurrentJITLocker locker(table->m_lock);
5434             for (auto iter = table->begin(locker), end = table->end(locker); iter != end; ++iter) {
5435                 bool found = false;
5436                 for (unsigned i = 0; i < data.m_properties.size(); ++i) {
5437                     if (iter->value.scopeOffset().offset() == data.m_properties[i].m_identifierNumber) {
5438                         found = true;
5439                         break;
5440                     }
5441                 }
5442                 ASSERT_UNUSED(found, found);
5443             }
5444         }
5445
5446         setJSValue(activation);
5447     }
5448
5449 #if ENABLE(FTL_NATIVE_CALL_INLINING)
5450     LValue getFunctionBySymbol(const CString symbol)
5451     {
5452         if (!m_ftlState.symbolTable.contains(symbol))
5453             return nullptr;
5454         if (!getModuleByPathForSymbol(m_ftlState.symbolTable.get(symbol), symbol))
5455             return nullptr;
5456         return getNamedFunction(m_ftlState.module, symbol.data());
5457     }
5458
5459     bool getModuleByPathForSymbol(const CString path, const CString symbol)
5460     {
5461         if (m_ftlState.nativeLoadedLibraries.contains(path)) {
5462             LValue function = getNamedFunction(m_ftlState.module, symbol.data());
5463             if (!isInlinableSize(function)) {
5464                 // We had no choice but to compile this function, but don't try to inline it ever again.
5465                 m_ftlState.symbolTable.remove(symbol);
5466                 return false;
5467             }
5468             return true;
5469         }
5470
5471         LMemoryBuffer memBuf;
5472
5473         ASSERT(isX86() || isARM64());
5474
5475 #if PLATFORM(EFL)
5476         const CString actualPath = toCString(bundlePath().data(), "/runtime/", path.data());
5477 #else
5478         const CString actualPath = toCString(bundlePath().data(),
5479             isX86() ? "/Resources/Runtime/x86_64/" : "/Resources/Runtime/arm64/",
5480             path.data());
5481 #endif
5482
5483         char* outMsg;
5484
5485         if (createMemoryBufferWithContentsOfFile(actualPath.data(), &memBuf, &outMsg)) {
5486             if (Options::verboseFTLFailure())
5487                 dataLog("Failed to load module at ", actualPath, "\n for symbol ", symbol, "\nERROR: ", outMsg, "\n");
5488             disposeMessage(outMsg);
5489             return false;
5490         }
5491
5492         LModule module;
5493
5494         if (parseBitcodeInContext(m_ftlState.context, memBuf, &module, &outMsg)) {
5495             if (Options::verboseFTLFailure())
5496                 dataLog("Failed to parse module at ", actualPath, "\n for symbol ", symbol, "\nERROR: ", outMsg, "\n");
5497             disposeMemoryBuffer(memBuf);
5498             disposeMessage(outMsg);
5499             return false;
5500         }
5501
5502         disposeMemoryBuffer(memBuf);
5503
5504         if (LValue function = getNamedFunction(m_ftlState.module, symbol.data())) {
5505             if (!isInlinableSize(function)) {
5506                 m_ftlState.symbolTable.remove(symbol);
5507                 disposeModule(module);
5508                 return false;
5509             }
5510         }
5511
5512         Vector<CString> namedFunctions;
5513         for (LValue function = getFirstFunction(module); function; function = getNextFunction(function)) {
5514             CString functionName(getValueName(function));
5515             namedFunctions.append(functionName);
5516
5517             for (LBasicBlock basicBlock = getFirstBasicBlock(function); basicBlock; basicBlock = getNextBasicBlock(basicBlock)) {
5518                 for (LValue instruction = getFirstInstruction(basicBlock); instruction; instruction = getNextInstruction(instruction)) {
5519                     setMetadata(instruction, m_tbaaKind, nullptr);
5520                     setMetadata(instruction, m_tbaaStructKind, nullptr);
5521                 }
5522             }
5523         }
5524
5525         Vector<CString> namedGlobals;
5526         for (LValue global = getFirstGlobal(module); global; global = getNextGlobal(global)) {
5527             CString globalName(getValueName(global));
5528             namedGlobals.append(globalName);
5529         }
5530
5531         if (linkModules(m_ftlState.module, module, LLVMLinkerDestroySource, &outMsg)) {
5532             if (Options::verboseFTLFailure())
5533                 dataLog("Failed to link module at ", actualPath, "\n for symbol ", symbol, "\nERROR: ", outMsg, "\n");
5534             disposeMessage(outMsg);
5535             return false;
5536         }
5537
5538         for (CString* symbol = namedFunctions.begin(); symbol != namedFunctions.end(); ++symbol) {
5539             LValue function = getNamedFunction(m_ftlState.module, symbol->data());
5540             LLVMLinkage linkage = getLinkage(function);
5541             if (linkage != LLVMInternalLinkage && linkage != LLVMPrivateLinkage)
5542                 setVisibility(function, LLVMHiddenVisibility);
5543             if (!isDeclaration(function)) {
5544                 setLinkage(function, LLVMPrivateLinkage);
5545                 setLinkage(function, LLVMAvailableExternallyLinkage);
5546
5547                 if (ASSERT_DISABLED)
5548                     removeFunctionAttr(function, LLVMStackProtectAttribute);
5549             }
5550         }
5551
5552         for (CString* symbol = namedGlobals.begin(); symbol != namedGlobals.end(); ++symbol) {
5553             LValue global = getNamedGlobal(m_ftlState.module, symbol->data());
5554             LLVMLinkage linkage = getLinkage(global);
5555             if (linkage != LLVMInternalLinkage && linkage != LLVMPrivateLinkage)
5556                 setVisibility(global, LLVMHiddenVisibility);
5557             if (!isDeclaration(global))
5558                 setLinkage(global, LLVMPrivateLinkage);
5559         }
5560
5561         m_ftlState.nativeLoadedLibraries.add(path);
5562         return true;
5563     }
5564 #endif
5565
5566     bool isInlinableSize(LValue function)
5567     {
5568         size_t instructionCount = 0;
5569         size_t maxSize = Options::maximumLLVMInstructionCountForNativeInlining();
5570         for (LBasicBlock basicBlock = getFirstBasicBlock(function); basicBlock; basicBlock = getNextBasicBlock(basicBlock)) {
5571             for (LValue instruction = getFirstInstruction(basicBlock); instruction; instruction = getNextInstruction(instruction)) {
5572                 if (++instructionCount >= maxSize)
5573                     return false;
5574             }
5575         }
5576         return true;
5577     }
5578
5579     LValue didOverflowStack()
5580     {
5581         // This does a very simple leaf function analysis. The invariant of FTL call
5582         // frames is that the caller had already done enough of a stack check to
5583         // prove that this call frame has enough stack to run, and also enough stack
5584         // to make runtime calls. So, we only need to stack check when making calls
5585         // to other JS functions. If we don't find such calls then we don't need to
5586         // do any stack checks.
5587
5588         for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
5589             BasicBlock* block = m_graph.block(blockIndex);
5590             if (!block)
5591                 continue;
5592
5593             for (unsigned nodeIndex = block->size(); nodeIndex--;) {
5594                 Node* node = block->at(nodeIndex);
5595
5596                 switch (node->op()) {
5597                 case GetById:
5598                 case PutById:
5599                 case Call:
5600                 case Construct:
5601                 case NativeCall:
5602                 case NativeConstruct:
5603                     return m_out.below(
5604                         m_callFrame,
5605                         m_out.loadPtr(
5606                             m_out.absolute(vm().addressOfFTLStackLimit())));
5607
5608                 default:
5609                     break;
5610                 }
5611             }
5612         }
5613
5614         return m_out.booleanFalse;
5615     }
5616
5617     struct ArgumentsLength {
5618         ArgumentsLength()
5619             : isKnown(false)
5620             , known(UINT_MAX)
5621             , value(nullptr)
5622         {
5623         }
5624
5625         bool isKnown;
5626         unsigned known;
5627         LValue value;
5628     };
5629     ArgumentsLength getArgumentsLength(InlineCallFrame* inlineCallFrame)
5630     {
5631         ArgumentsLength length;
5632
5633         if (inlineCallFrame && !inlineCallFrame->isVarargs()) {
5634             length.known = inlineCallFrame->arguments.size() - 1;
5635             length.isKnown = true;
5636             length.value = m_out.constInt32(length.known);
5637         } else {
5638             length.known = UINT_MAX;
5639             length.isKnown = false;
5640
5641             VirtualRegister argumentCountRegister;
5642             if (!inlineCallFrame)
5643                 argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
5644             else
5645                 argumentCountRegister = inlineCallFrame->argumentCountRegister;
5646             length.value = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
5647         }
5648
5649         return length;
5650     }
5651
5652     ArgumentsLength getArgumentsLength()
5653     {
5654         return getArgumentsLength(m_node->origin.semantic.inlineCallFrame);
5655     }
5656
5657     LValue getCurrentCallee()
5658     {
5659         if (InlineCallFrame* frame = m_node->origin.semantic.inlineCallFrame) {
5660             if (frame->isClosureCall)
5661                 return m_out.loadPtr(addressFor(frame->calleeRecovery.virtualRegister()));
5662             return weakPointer(frame->calleeRecovery.constant().asCell());
5663         }
5664         return m_out.loadPtr(addressFor(JSStack::Callee));
5665     }
5666
5667     LValue getArgumentsStart(InlineCallFrame* inlineCallFrame)
5668     {
5669         VirtualRegister start = AssemblyHelpers::argumentsStart(inlineCallFrame);
5670         return addressFor(start).value();
5671     }
5672
5673     LValue getArgumentsStart()
5674     {
5675         return getArgumentsStart(m_node->origin.semantic.inlineCallFrame);
5676     }
5677
5678     template<typename Functor>
5679     void checkStructure(
5680         LValue structureDiscriminant, const FormattedValue& formattedValue, ExitKind exitKind,
5681         const StructureSet& set, const Functor& weakStructureDiscriminant)
5682     {
5683         if (set.size() == 1) {
5684             speculate(
5685                 exitKind, formattedValue, 0,
5686                 m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set[0])));
5687             return;
5688         }
5689
5690         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("checkStructure continuation"));
5691
5692         LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
5693         for (unsigned i = 0; i < set.size() - 1; ++i) {
5694             LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("checkStructure nextStructure"));
5695             m_out.branch(
5696                 m_out.equal(structureDiscriminant, weakStructureDiscriminant(set[i])),
5697                 unsure(continuation), unsure(nextStructure));
5698             m_out.appendTo(nextStructure);
5699         }
5700
5701         speculate(
5702             exitKind, formattedValue, 0,
5703             m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set.last())));
5704
5705         m_out.jump(continuation);
5706         m_out.appendTo(continuation, lastNext);
5707     }
5708
5709     LValue numberOrNotCellToInt32(Edge edge, LValue value)
5710     {
5711         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 int case"));
5712         LBasicBlock notIntCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not int case"));
5713         LBasicBlock doubleCase = 0;
5714         LBasicBlock notNumberCase = 0;
5715         if (edge.useKind() == NotCellUse) {
5716             doubleCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 double case"));
5717             notNumberCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not number case"));
5718         }
5719         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ValueToInt32 continuation"));
5720
5721         Vector<ValueFromBlock> results;
5722
5723         m_out.branch(isNotInt32(value), unsure(notIntCase), unsure(intCase));
5724
5725         LBasicBlock lastNext = m_out.appendTo(intCase, notIntCase);
5726         results.append(m_out.anchor(unboxInt32(value)));
5727         m_out.jump(continuation);
5728
5729         if (edge.useKind() == NumberUse) {
5730             m_out.appendTo(notIntCase, continuation);
5731             FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isCellOrMisc(value));
5732             results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
5733             m_out.jump(continuation);
5734         } else {
5735             m_out.appendTo(notIntCase, doubleCase);
5736             m_out.branch(
5737                 isCellOrMisc(value, provenType(edge)), unsure(notNumberCase), unsure(doubleCase));
5738
5739             m_out.appendTo(doubleCase, notNumberCase);
5740             results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
5741             m_out.jump(continuation);
5742
5743             m_out.appendTo(notNumberCase, continuation);
5744
5745             FTL_TYPE_CHECK(jsValueValue(value), edge, ~SpecCell, isCell(value));
5746
5747             LValue specialResult = m_out.select(
5748                 m_out.equal(value, m_out.constInt64(JSValue::encode(jsBoolean(true)))),
5749                 m_out.int32One, m_out.int32Zero);
5750             results.append(m_out.anchor(specialResult));
5751             m_out.jump(continuation);
5752         }
5753
5754         m_out.appendTo(continuation, lastNext);
5755         return m_out.phi(m_out.int32, results);
5756     }
5757
5758     LValue loadProperty(LValue storage, unsigned identifierNumber, PropertyOffset offset)
5759     {
5760         return m_out.load64(addressOfProperty(storage, identifierNumber, offset));
5761     }
5762
5763     void storeProperty(
5764         LValue value, LValue storage, unsigned identifierNumber, PropertyOffset offset)
5765     {
5766         m_out.store64(value, addressOfProperty(storage, identifierNumber, offset));
5767     }
5768
5769     TypedPointer addressOfProperty(
5770         LValue storage, unsigned identifierNumber, PropertyOffset offset)
5771     {
5772         return m_out.address(
5773             m_heaps.properties[identifierNumber], storage, offsetRelativeToBase(offset));
5774     }
5775
5776     LValue storageForTransition(
5777         LValue object, PropertyOffset offset,
5778         Structure* previousStructure, Structure* nextStructure)
5779     {
5780         if (isInlineOffset(offset))
5781             return object;
5782
5783         if (previousStructure->outOfLineCapacity() == nextStructure->outOfLineCapacity())
5784             return m_out.loadPtr(object, m_heaps.JSObject_butterfly);
5785
5786         LValue result;
5787         if (!previousStructure->outOfLineCapacity())
5788             result = allocatePropertyStorage(object, previousStructure);
5789         else {
5790             result = reallocatePropertyStorage(
5791                 object, m_out.loadPtr(object, m_heaps.JSObject_butterfly),
5792                 previousStructure, nextStructure);
5793         }
5794
5795         emitStoreBarrier(object);
5796
5797         return result;
5798     }
5799
5800     LValue allocatePropertyStorage(LValue object, Structure* previousStructure)
5801     {
5802         if (previousStructure->couldHaveIndexingHeader()) {
5803             return vmCall(
5804                 m_out.operation(
5805                     operationReallocateButterflyToHavePropertyStorageWithInitialCapacity),
5806                 m_callFrame, object);
5807         }
5808
5809         LValue result = allocatePropertyStorageWithSizeImpl(initialOutOfLineCapacity);
5810         m_out.storePtr(result, object, m_heaps.JSObject_butterfly);
5811         return result;
5812     }
5813
5814     LValue reallocatePropertyStorage(
5815         LValue object, LValue oldStorage, Structure* previous, Structure* next)
5816     {
5817         size_t oldSize = previous->outOfLineCapacity();
5818         size_t newSize = oldSize * outOfLineGrowthFactor;
5819
5820         ASSERT_UNUSED(next, newSize == next->outOfLineCapacity());
5821
5822         if (previous->couldHaveIndexingHeader()) {
5823             LValue newAllocSize = m_out.constIntPtr(newSize);
5824             return vmCall(m_out.operation(operationReallocateButterflyToGrowPropertyStorage), m_callFrame, object, newAllocSize);
5825         }
5826
5827         LValue result = allocatePropertyStorageWithSizeImpl(newSize);
5828
5829         ptrdiff_t headerSize = -sizeof(IndexingHeader) - sizeof(void*);
5830         ptrdiff_t endStorage = headerSize - static_cast<ptrdiff_t>(oldSize * sizeof(JSValue));
5831
5832         for (ptrdiff_t offset = headerSize; offset > endStorage; offset -= sizeof(void*)) {
5833             LValue loaded =
5834                 m_out.loadPtr(m_out.address(m_heaps.properties.atAnyNumber(), oldStorage, offset));
5835             m_out.storePtr(loaded, m_out.address(m_heaps.properties.atAnyNumber(), result, offset));
5836         }
5837
5838         m_out.storePtr(result, m_out.address(object, m_heaps.JSObject_butterfly));
5839
5840         return result;
5841     }
5842
5843     LValue allocatePropertyStorageWithSizeImpl(size_t sizeInValues)
5844     {
5845         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorageWithSizeImpl slow path"));
5846         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorageWithSizeImpl continuation"));
5847
5848         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5849
5850         LValue endOfStorage = allocateBasicStorageAndGetEnd(
5851             m_out.constIntPtr(sizeInValues * sizeof(JSValue)), slowPath);
5852
5853         ValueFromBlock fastButterfly = m_out.anchor(
5854             m_out.add(m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage));
5855
5856         m_out.jump(continuation);
5857
5858         m_out.appendTo(slowPath, continuation);
5859
5860         LValue slowButterflyValue;
5861         if (sizeInValues == initialOutOfLineCapacity) {
5862             slowButterflyValue = vmCall(
5863                 m_out.operation(operationAllocatePropertyStorageWithInitialCapacity),
5864                 m_callFrame);
5865         } else {
5866             slowButterflyValue = vmCall(
5867                 m_out.operation(operationAllocatePropertyStorage),
5868                 m_callFrame, m_out.constIntPtr(sizeInValues));
5869         }
5870         ValueFromBlock slowButterfly = m_out.anchor(slowButterflyValue);
5871
5872         m_out.jump(continuation);
5873
5874         m_out.appendTo(continuation, lastNext);
5875
5876         return m_out.phi(m_out.intPtr, fastButterfly, slowButterfly);
5877     }
5878
5879     LValue getById(LValue base)
5880     {
5881         auto uid = m_graph.identifiers()[m_node->identifierNumber()];
5882
5883         // Arguments: id, bytes, target, numArgs, args...
5884         unsigned stackmapID = m_stackmapIDs++;
5885
5886         if (Options::verboseCompilation())
5887             dataLog("    Emitting GetById patchpoint with stackmap #", stackmapID, "\n");
5888
5889         LValue call = m_out.call(
5890             m_out.patchpointInt64Intrinsic(),
5891             m_out.constInt64(stackmapID), m_out.constInt32(sizeOfGetById()),
5892             constNull(m_out.ref8), m_out.constInt32(1), base);
5893         setInstructionCallingConvention(call, LLVMAnyRegCallConv);
5894
5895         m_ftlState.getByIds.append(GetByIdDescriptor(stackmapID, m_node->origin.semantic, uid));
5896
5897         return call;
5898     }
5899
5900     TypedPointer baseIndex(IndexedAbstractHeap& heap, LValue storage, LValue index, Edge edge, ptrdiff_t offset = 0)
5901     {
5902         return m_out.baseIndex(
5903             heap, storage, m_out.zeroExtPtr(index), provenValue(edge), offset);
5904     }
5905
5906     void compare(
5907         LIntPredicate intCondition, LRealPredicate realCondition,
5908         S_JITOperation_EJJ helperFunction)
5909     {
5910         if (m_node->isBinaryUseKind(Int32Use)) {
5911             LValue left = lowInt32(m_node->child1());
5912             LValue right = lowInt32(m_node->child2());
5913             setBoolean(m_out.icmp(intCondition, left, right));
5914             return;
5915         }
5916
5917         if (m_node->isBinaryUseKind(Int52RepUse)) {
5918             Int52Kind kind;
5919             LValue left = lowWhicheverInt52(m_node->child1(), kind);
5920             LValue right = lowInt52(m_node->child2(), kind);
5921             setBoolean(m_out.icmp(intCondition, left, right));
5922             return;
5923         }
5924
5925         if (m_node->isBinaryUseKind(DoubleRepUse)) {
5926             LValue left = lowDouble(m_node->child1());
5927             LValue right = lowDouble(m_node->child2());
5928             setBoolean(m_out.fcmp(realCondition, left, right));
5929             return;
5930         }
5931
5932         if (m_node->isBinaryUseKind(UntypedUse)) {
5933             nonSpeculativeCompare(intCondition, helperFunction);
5934             return;
5935         }
5936
5937         DFG_CRASH(m_graph, m_node, "Bad use kinds");
5938     }
5939
5940     void compareEqObjectOrOtherToObject(Edge leftChild, Edge rightChild)
5941     {
5942         LValue rightCell = lowCell(rightChild);
5943         LValue leftValue = lowJSValue(leftChild, ManualOperandSpeculation);
5944
5945         speculateTruthyObject(rightChild, rightCell, SpecObject);
5946
5947         LBasicBlock leftCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left cell case"));
5948         LBasicBlock leftNotCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left not cell case"));
5949         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject continuation"));
5950
5951         m_out.branch(
5952             isCell(leftValue, provenType(leftChild)),
5953             unsure(leftCellCase), unsure(leftNotCellCase));
5954
5955         LBasicBlock lastNext = m_out.appendTo(leftCellCase, leftNotCellCase);
5956         speculateTruthyObject(leftChild, leftValue, SpecObject | (~SpecCell));
5957         ValueFromBlock cellResult = m_out.anchor(m_out.equal(rightCell, leftValue));
5958         m_out.jump(continuation);
5959
5960         m_out.appendTo(leftNotCellCase, continuation);
5961         FTL_TYPE_CHECK(
5962             jsValueValue(leftValue), leftChild, SpecOther | SpecCell, isNotOther(leftValue));
5963         ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
5964         m_out.jump(continuation);
5965
5966         m_out.appendTo(continuation, lastNext);
5967         setBoolean(m_out.phi(m_out.boolean, cellResult, notCellResult));
5968     }
5969
5970     void speculateTruthyObject(Edge edge, LValue cell, SpeculatedType filter)
5971     {
5972         if (masqueradesAsUndefinedWatchpointIsStillValid()) {
5973             FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell));
5974             return;
5975         }
5976
5977         FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell));
5978         speculate(
5979             BadType, jsValueValue(cell), edge.node(),
5980             m_out.testNonZero8(
5981                 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
5982                 m_out.constInt8(MasqueradesAsUndefined)));
5983     }
5984
5985     void nonSpeculativeCompare(LIntPredicate intCondition, S_JITOperation_EJJ helperFunction)
5986     {
5987         LValue left = lowJSValue(m_node->child1());
5988         LValue right = lowJSValue(m_node->child2());
5989
5990         LBasicBlock leftIsInt = FTL_NEW_BLOCK(m_out, ("CompareEq untyped left is int"));
5991         LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped fast path"));
5992         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped slow path"));
5993         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEq untyped continuation"));
5994
5995         m_out.branch(isNotInt32(left), rarely(slowPath), usually(leftIsInt));
5996
5997         LBasicBlock lastNext = m_out.appendTo(leftIsInt, fastPath);
5998         m_out.branch(isNotInt32(right), rarely(slowPath), usually(fastPath));
5999
6000         m_out.appendTo(fastPath, slowPath);
6001         ValueFromBlock fastResult = m_out.anchor(
6002             m_out.icmp(intCondition, unboxInt32(left), unboxInt32(right)));
6003         m_out.jump(continuation);
6004
6005         m_out.appendTo(slowPath, continuation);
6006         ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall(
6007             m_out.operation(helperFunction), m_callFrame, left, right)));
6008         m_out.jump(continuation);
6009
6010         m_out.appendTo(continuation, lastNext);
6011         setBoolean(m_out.phi(m_out.boolean, fastResult, slowResult));
6012     }
6013
6014     LValue allocateCell(LValue allocator, LBasicBlock slowPath)
6015     {
6016         LBasicBlock success = FTL_NEW_BLOCK(m_out, ("object allocation success"));
6017
6018         LValue result = m_out.loadPtr(
6019             allocator, m_heaps.MarkedAllocator_freeListHead);
6020
6021         m_out.branch(m_out.notNull(result), usually(success), rarely(slowPath));
6022
6023         m_out.appendTo(success);
6024
6025         m_out.storePtr(
6026             m_out.loadPtr(result, m_heaps.JSCell_freeListNext),
6027             allocator, m_heaps.MarkedAllocator_freeListHead);
6028
6029         return result;
6030     }
6031
6032     void storeStructure(LValue object, Structure* structure)
6033     {
6034         m_out.store32(m_out.constInt32(structure->id()), object, m_heaps.JSCell_structureID);
6035         m_out.store32(
6036             m_out.constInt32(structure->objectInitializationBlob()),
6037             object, m_heaps.JSCell_usefulBytes);
6038     }
6039
6040     LValue allocateCell(LValue allocator, Structure* structure, LBasicBlock slowPath)
6041     {
6042         LValue result = allocateCell(allocator, slowPath);
6043         storeStructure(result, structure);
6044         return result;
6045     }
6046
6047     LValue allocateObject(
6048         LValue allocator, Structure* structure, LValue butterfly, LBasicBlock slowPath)
6049     {
6050         LValue result = allocateCell(allocator, structure, slowPath);
6051         m_out.storePtr(butterfly, result, m_heaps.JSObject_butterfly);
6052         return result;
6053     }
6054
6055     template<typename ClassType>
6056     LValue allocateObject(
6057         size_t size, Structure* structure, LValue butterfly, LBasicBlock slowPath)
6058     {
6059         MarkedAllocator* allocator = &vm().heap.allocatorForObjectOfType<ClassType>(size);
6060         return allocateObject(m_out.constIntPtr(allocator), structure, butterfly, slowPath);
6061     }
6062
6063     template<typename ClassType>
6064     LValue allocateObject(Structure* structure, LValue butterfly, LBasicBlock slowPath)
6065     {
6066         return allocateObject<ClassType>(
6067             ClassType::allocationSize(0), structure, butterfly, slowPath);
6068     }
6069
6070     template<typename ClassType>
6071     LValue allocateVariableSizedObject(
6072         LValue size, Structure* structure, LValue butterfly, LBasicBlock slowPath)
6073     {
6074         static_assert(!(MarkedSpace::preciseStep & (MarkedSpace::preciseStep - 1)), "MarkedSpace::preciseStep must be a power of two.");
6075         static_assert(!(MarkedSpace::impreciseStep & (MarkedSpace::impreciseStep - 1)), "MarkedSpace::impreciseStep must be a power of two.");
6076
6077         LValue subspace = m_out.constIntPtr(&vm().heap.subspaceForObjectOfType<ClassType>());
6078
6079         LBasicBlock smallCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject small case"));
6080         LBasicBlock largeOrOversizeCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject large or oversize case"));
6081         LBasicBlock largeCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject large case"));
6082         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject continuation"));
6083
6084         LValue uproundedSize = m_out.add(size, m_out.constInt32(MarkedSpace::preciseStep - 1));
6085         LValue isSmall = m_out.below(uproundedSize, m_out.constInt32(MarkedSpace::preciseCutoff));
6086         m_out.branch(isSmall, unsure(smallCaseBlock), unsure(largeOrOversizeCaseBlock));
6087
6088         LBasicBlock lastNext = m_out.appendTo(smallCaseBlock, largeOrOversizeCaseBlock);
6089         TypedPointer address = m_out.baseIndex(
6090             m_heaps.MarkedSpace_Subspace_preciseAllocators, subspace,
6091             m_out.zeroExtPtr(m_out.lShr(uproundedSize, m_out.constInt32(getLSBSet(MarkedSpace::preciseStep)))));
6092         ValueFromBlock smallAllocator = m_out.anchor(address.value());
6093         m_out.jump(continuation);
6094
6095         m_out.appendTo(largeOrOversizeCaseBlock, largeCaseBlock);
6096         m_out.branch(
6097             m_out.below(uproundedSize, m_out.constInt32(MarkedSpace::impreciseCutoff)),
6098             usually(largeCaseBlock), rarely(slowPath));
6099
6100         m_out.appendTo(largeCaseBlock, continuation);
6101         address = m_out.baseIndex(
6102             m_heaps.MarkedSpace_Subspace_impreciseAllocators, subspace,
6103             m_out.zeroExtPtr(m_out.lShr(uproundedSize, m_out.constInt32(getLSBSet(MarkedSpace::impreciseStep)))));
6104         ValueFromBlock largeAllocator = m_out.anchor(address.value());
6105         m_out.jump(continuation);
6106
6107         m_out.appendTo(continuation, lastNext);
6108         LValue allocator = m_out.phi(m_out.intPtr, smallAllocator, largeAllocator);
6109
6110         return allocateObject(allocator, structure, butterfly, slowPath);
6111     }
6112
6113     // Returns a pointer to the end of the allocation.
6114     LValue allocateBasicStorageAndGetEnd(LValue size, LBasicBlock slowPath)
6115     {
6116         CopiedAllocator& allocator = vm().heap.storageAllocator();
6117
6118         LBasicBlock success = FTL_NEW_BLOCK(m_out, ("storage allocation success"));
6119
6120         LValue remaining = m_out.loadPtr(m_out.absolute(&allocator.m_currentRemaining));
6121         LValue newRemaining = m_out.sub(remaining, size);
6122
6123         m_out.branch(
6124             m_out.lessThan(newRemaining, m_out.intPtrZero),
6125             rarely(slowPath), usually(success));
6126
6127         m_out.appendTo(success);
6128
6129         m_out.storePtr(newRemaining, m_out.absolute(&allocator.m_currentRemaining));
6130         return m_out.sub(
6131             m_out.loadPtr(m_out.absolute(&allocator.m_currentPayloadEnd)), newRemaining);
6132     }
6133
6134     LValue allocateObject(Structure* structure)
6135     {
6136         size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
6137         MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize);
6138
6139         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("allocateObject slow path"));
6140         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocateObject continuation"));
6141
6142         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
6143
6144         ValueFromBlock fastResult = m_out.anchor(allocateObject(
6145             m_out.constIntPtr(allocator), structure, m_out.intPtrZero, slowPath));
6146
6147         m_out.jump(continuation);
6148
6149         m_out.appendTo(slowPath, continuation);
6150
6151         ValueFromBlock slowResult = m_out.anchor(vmCall(
6152             m_out.operation(operationNewObject), m_callFrame, m_out.constIntPtr(structure)));
6153         m_out.jump(continuation);
6154
6155         m_out.appendTo(continuation, lastNext);
6156         return m_out.phi(m_out.intPtr, fastResult, slowResult);
6157     }
6158
6159     struct ArrayValues {
6160         ArrayValues()
6161             : array(0)
6162             , butterfly(0)
6163         {
6164         }
6165
6166         ArrayValues(LValue array, LValue butterfly)
6167             : array(array)
6168             , butterfly(butterfly)
6169         {
6170         }
6171
6172         LValue array;
6173         LValue butterfly;
6174     };
6175     ArrayValues allocateJSArray(
6176         Structure* structure, unsigned numElements, LBasicBlock slowPath)
6177     {
6178         ASSERT(
6179             hasUndecided(structure->indexingType())
6180             || hasInt32(structure->indexingType())
6181             || hasDouble(structure->indexingType())
6182             || hasContiguous(structure->indexingType()));
6183
6184         unsigned vectorLength = std::max(BASE_VECTOR_LEN, numElements);
6185
6186         LValue endOfStorage = allocateBasicStorageAndGetEnd(
6187             m_out.constIntPtr(sizeof(JSValue) * vectorLength + sizeof(IndexingHeader)),
6188             slowPath);
6189
6190         LValue butterfly = m_out.sub(
6191             endOfStorage, m_out.constIntPtr(sizeof(JSValue) * vectorLength));
6192
6193         LValue object = allocateObject<JSArray>(
6194             structure, butterfly, slowPath);
6195
6196         m_out.store32(m_out.constInt32(numElements), butterfly, m_heaps.Butterfly_publicLength);
6197         m_out.store32(m_out.constInt32(vectorLength), butterfly, m_heaps.Butterfly_vectorLength);
6198
6199         if (hasDouble(structure->indexingType())) {
6200             for (unsigned i = numElements; i < vectorLength; ++i) {
6201                 m_out.store64(
6202                     m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
6203                     butterfly, m_heaps.indexedDoubleProperties[i]);
6204             }
6205         }
6206
6207         return ArrayValues(object, butterfly);
6208     }
6209
6210     ArrayValues allocateJSArray(Structure* structure, unsigned numElements)
6211     {
6212         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("JSArray allocation slow path"));
6213         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("JSArray allocation continuation"));
6214
6215         LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
6216
6217         ArrayValues fastValues = allocateJSArray(structure, numElements, slowPath);
6218         ValueFromBlock fastArray = m_out.anchor(fastValues.array);
6219         ValueFromBlock fastButterfly = m_out.anchor(fastValues.butterfly);
6220
6221         m_out.jump(continuation);
6222
6223         m_out.appendTo(slowPath, continuation);
6224
6225         ValueFromBlock slowArray = m_out.anchor(vmCall(
6226             m_out.operation(operationNewArrayWithSize), m_callFrame,
6227             m_out.constIntPtr(structure), m_out.constInt32(numElements)));
6228         ValueFromBlock slowButterfly = m_out.anchor(
6229             m_out.loadPtr(slowArray.value(), m_heaps.JSObject_butterfly));
6230
6231         m_out.jump(continuation);
6232
6233         m_out.appendTo(continuation, lastNext);
6234
6235         return ArrayValues(
6236             m_out.phi(m_out.intPtr, fastArray, slowArray),
6237             m_out.phi(m_out.intPtr, fastButterfly, slowButterfly));
6238     }
6239
6240     LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode, LValue base)
6241     {
6242         JSArrayBufferView* view = m_graph.tryGetFoldableView(provenValue(baseEdge), arrayMode);
6243         if (view)
6244             return m_out.constInt32(view->length());
6245         return m_out.load32NonNegative(base, m_heaps.JSArrayBufferView_length);
6246     }
6247
6248     LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode)
6249     {
6250         return typedArrayLength(baseEdge, arrayMode, lowCell(baseEdge));
6251     }
6252
6253     LValue boolify(Edge edge)
6254     {
6255         switch (edge.useKind()) {
6256         case BooleanUse:
6257             return lowBoolean(edge);
6258         case Int32Use:
6259             return m_out.notZero32(lowInt32(edge));
6260         case DoubleRepUse:
6261             return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
6262         case ObjectOrOtherUse:
6263             return m_out.bitNot(
6264                 equalNullOrUndefined(
6265                     edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
6266                     ManualOperandSpeculation));
6267         case StringUse: {
6268             LValue stringValue = lowString(edge);
6269             LValue length = m_out.load32NonNegative(stringValue, m_heaps.JSString_length);
6270             return m_out.notEqual(length, m_out.int32Zero);
6271         }
6272         case UntypedUse: {
6273             LValue value = lowJSValue(edge);
6274
6275             // Implements the following control flow structure:
6276             // if (value is cell) {
6277             //     if (value is string)
6278             //         result = !!value->length
6279             //     else {
6280             //         do evil things for masquerades-as-undefined
6281             //         result = true
6282             //     }
6283             // } else if (value is int32) {
6284             //     result = !!unboxInt32(value)
6285             // } else if (value is number) {
6286             //     result = !!unboxDouble(value)
6287             // } else {
6288             //     result = value == jsTrue
6289             // }
6290
6291             LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped cell case"));
6292             LBasicBlock stringCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped string case"));
6293             LBasicBlock notStringCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not string case"));
6294             LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not cell case"));
6295             LBasicBlock int32Case = FTL_NEW_BLOCK(m_out, ("Boolify untyped int32 case"));
6296             LBasicBlock notInt32Case = FTL_NEW_BLOCK(m_out, ("Boolify untyped not int32 case"));
6297             LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped double case"));
6298             LBasicBlock notDoubleCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not double case"));
6299             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Boolify untyped continuation"));
6300
6301             Vector<ValueFromBlock> results;
6302
6303             m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase));
6304
6305             LBasicBlock lastNext = m_out.appendTo(cellCase, stringCase);
6306             m_out.branch(
6307                 isString(value, provenType(edge) & SpecCell),
6308                 unsure(stringCase), unsure(notStringCase));
6309
6310             m_out.appendTo(stringCase, notStringCase);
6311             LValue nonEmptyString = m_out.notZero32(
6312                 m_out.load32NonNegative(value, m_heaps.JSString_length));
6313             results.append(m_out.anchor(nonEmptyString));
6314             m_out.jump(continuation);
6315
6316             m_out.appendTo(notStringCase, notCellCase);
6317             LValue isTruthyObject;
6318             if (masqueradesAsUndefinedWatchpointIsStillValid())
6319                 isTruthyObject = m_out.booleanTrue;
6320             else {
6321                 LBasicBlock masqueradesCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped masquerades case"));
6322
6323                 results.append(m_out.anchor(m_out.booleanTrue));
6324
6325                 m_out.branch(
6326                     m_out.testIsZero8(
6327                         m_out.load8(value, m_heaps.JSCell_typeInfoFlags),
6328                         m_out.constInt8(MasqueradesAsUndefined)),
6329                     usually(continuation), rarely(masqueradesCase));
6330
6331                 m_out.appendTo(masqueradesCase);
6332
6333                 isTruthyObject = m_out.notEqual(
6334                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->origin.semantic)),
6335                     m_out.loadPtr(loadStructure(value), m_heaps.Structure_globalObject));
6336             }
6337             results.append(m_out.anchor(isTruthyObject));
6338             m_out.jump(continuation);
6339
6340             m_out.appendTo(notCellCase, int32Case);
6341             m_out.branch(
6342                 isInt32(value, provenType(edge) & ~SpecCell),
6343                 unsure(int32Case), unsure(notInt32Case));
6344
6345             m_out.appendTo(int32Case, notInt32Case);
6346             results.append(m_out.anchor(m_out.notZero32(unboxInt32(value))));
6347             m_out.jump(continuation);
6348
6349             m_out.appendTo(notInt32Case, doubleCase);
6350             m_out.branch(
6351                 isNumber(value, provenType(edge) & ~SpecCell),
6352                 unsure(doubleCase), unsure(notDoubleCase));
6353
6354             m_out.appendTo(doubleCase, notDoubleCase);
6355             // Note that doubleNotEqual() really means not-equal-and-ordered. It will return false
6356             // if value is NaN.
6357             LValue doubleIsTruthy = m_out.doubleNotEqual(
6358                 unboxDouble(value), m_out.constDouble(0));
6359             results.append(m_out.anchor(doubleIsTruthy));
6360             m_out.jump(continuation);
6361
6362             m_out.appendTo(notDoubleCase, continuation);
6363             LValue miscIsTruthy = m_out.equal(
6364                 value, m_out.constInt64(JSValue::encode(jsBoolean(true))));
6365             results.append(m_out.anchor(miscIsTruthy));
6366             m_out.jump(continuation);
6367
6368             m_out.appendTo(continuation, lastNext);
6369             return m_out.phi(m_out.boolean, results);
6370         }
6371         default:
6372             DFG_CRASH(m_graph, m_node, "Bad use kind");
6373             return 0;
6374         }
6375     }
6376
6377     enum StringOrObjectMode {
6378         AllCellsAreFalse,
6379         CellCaseSpeculatesObject
6380     };
6381     enum EqualNullOrUndefinedMode {
6382         EqualNull,
6383         EqualUndefined,
6384         EqualNullOrUndefined,
6385         SpeculateNullOrUndefined
6386     };
6387     LValue equalNullOrUndefined(
6388         Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
6389         OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
6390     {
6391         bool validWatchpoint = masqueradesAsUndefinedWatchpointIsStillValid();
6392
6393         LValue value = lowJSValue(edge, operandMode);
6394
6395         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
6396         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
6397         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
6398
6399         m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase));
6400
6401         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
6402
6403         Vector<ValueFromBlock, 3> results;
6404
6405         switch (cellMode) {
6406         case AllCellsAreFalse:
6407             break;
6408         case CellCaseSpeculatesObject:
6409             FTL_TYPE_CHECK(
6410                 jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value));
6411             break;
6412         }
6413
6414         if (validWatchpoint) {
6415             results.append(m_out.anchor(m_out.booleanFalse));
6416             m_out.jump(continuation);
6417         } else {
6418             LBasicBlock masqueradesCase =
6419                 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
6420
6421             results.append(m_out.anchor(m_out.booleanFalse));
6422
6423             m_out.branch(
6424                 m_out.testNonZero8(
6425                     m_out.load8(value, m_heaps.JSCell_typeInfoFlags),
6426                     m_out.constInt8(MasqueradesAsUndefined)),
6427                 rarely(masqueradesCase), usually(continuation));
6428
6429             m_out.appendTo(masqueradesCase, primitiveCase);
6430
6431             LValue structure = loadStructure(value);
6432
6433             results.append(m_out.anchor(
6434                 m_out.equal(
6435                     m_out.constIntPtr(m_graph.globalObjectFor(m_node->origin.semantic)),
6436                     m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
6437             m_out.jump(continuation);
6438         }
6439
6440         m_out.appendTo(primitiveCase, continuation);
6441
6442         LValue primitiveResult;
6443         switch (primitiveMode) {
6444         case EqualNull:
6445             primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
6446             break;
6447         case EqualUndefined:
6448             primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
6449             break;
6450         case EqualNullOrUndefined:
6451             primitiveResult = isOther(value, provenType(edge));
6452             break;
6453         case SpeculateNullOrUndefined:
6454             FTL_TYPE_CHECK(
6455                 jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value));
6456             primitiveResult = m_out.booleanTrue;
6457             break;
6458         }
6459         results.append(m_out.anchor(primitiveResult));
6460         m_out.jump(continuation);
6461
6462         m_out.appendTo(continuation, lastNext);
6463
6464         return m_out.phi(m_out.boolean, results);
6465     }
6466
6467     template<typename FunctionType>
6468     void contiguousPutByValOutOfBounds(
6469         FunctionType slowPathFunction, LValue base, LValue storage, LValue index, LValue value,
6470         LBasicBlock continuation)
6471     {
6472         LValue isNotInBounds = m_out.aboveOrEqual(
6473             index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength));
6474         if (!m_node->arrayMode().isInBounds()) {
6475             LBasicBlock notInBoundsCase =
6476                 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
6477             LBasicBlock performStore =
6478                 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
6479
6480             m_out.branch(isNotInBounds, unsure(notInBoundsCase), unsure(performStore));
6481
6482             LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
6483
6484             LValue isOutOfBounds = m_out.aboveOrEqual(
6485                 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_vectorLength));
6486
6487             if (!m_node->arrayMode().isOutOfBounds())
6488                 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
6489             else {
6490                 LBasicBlock outOfBoundsCase =
6491                     FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
6492                 LBasicBlock holeCase =
6493                     FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
6494
6495                 m_out.branch(isOutOfBounds, unsure(outOfBoundsCase), unsure(holeCase));
6496
6497                 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
6498
6499                 vmCall(
6500                     m_out.operation(slowPathFunction),
6501                     m_callFrame, base, index, value);
6502
6503                 m_out.jump(continuation);
6504
6505                 m_out.appendTo(holeCase, innerLastNext);
6506             }
6507
6508             m_out.store32(
6509                 m_out.add(index, m_out.int32One),
6510                 storage, m_heaps.Butterfly_publicLength);
6511
6512             m_out.jump(performStore);
6513             m_out.appendTo(performStore, lastNext);
6514         }
6515     }
6516
6517     void buildSwitch(SwitchData* data, LType type, LValue switchValue)
6518     {
6519         Vector<SwitchCase> cases;
6520         for (unsigned i = 0; i < data->cases.size(); ++i) {
6521             cases.append(SwitchCase(
6522                 constInt(type, data->cases[i].value.switchLookupValue(data->kind)),
6523                 lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count)));
6524         }
6525
6526         m_out.switchInstruction(
6527             switchValue, cases,
6528             lowBlock(data->fallThrough.block), Weight(data->fallThrough.count));
6529     }
6530
6531     void switchString(SwitchData* data, LValue string)
6532     {
6533         bool canDoBinarySwitch = true;
6534         unsigned totalLength = 0;
6535
6536         for (DFG::SwitchCase myCase : data->cases) {
6537             StringImpl* string = myCase.value.stringImpl();
6538             if (!string->is8Bit()) {
6539                 canDoBinarySwitch = false;
6540                 break;
6541             }
6542             if (string->length() > Options::maximumBinaryStringSwitchCaseLength()) {
6543                 canDoBinarySwitch = false;
6544                 break;
6545             }
6546             totalLength += string->length();
6547         }
6548
6549         if (!canDoBinarySwitch || totalLength > Options::maximumBinaryStringSwitchTotalLength()) {
6550             switchStringSlow(data, string);
6551             return;
6552         }
6553
6554         LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
6555         LValue length = m_out.load32(string, m_heaps.JSString_length);
6556
6557         LBasicBlock hasImplBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString has impl case"));
6558         LBasicBlock is8BitBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString is 8 bit case"));
6559         LBasicBlock slowBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString slow case"));
6560
6561         m_out.branch(m_out.isNull(stringImpl), unsure(slowBlock), unsure(hasImplBlock));
6562
6563         LBasicBlock lastNext = m_out.appendTo(hasImplBlock, is8BitBlock);
6564
6565         m_out.branch(
6566             m_out.testIsZero32(
6567                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
6568                 m_out.constInt32(StringImpl::flagIs8Bit())),
6569             unsure(slowBlock), unsure(is8BitBlock));
6570
6571         m_out.appendTo(is8BitBlock, slowBlock);
6572
6573         LValue buffer = m_out.loadPtr(stringImpl, m_heaps.StringImpl_data);
6574
6575         // FIXME: We should propagate branch weight data to the cases of this switch.
6576         // https://bugs.webkit.org/show_bug.cgi?id=144368
6577
6578         Vector<StringSwitchCase> cases;
6579         for (DFG::SwitchCase myCase : data->cases)
6580             cases.append(StringSwitchCase(myCase.value.stringImpl(), lowBlock(myCase.target.block)));
6581         std::sort(cases.begin(), cases.end());
6582         switchStringRecurse(data, buffer, length, cases, 0, 0, cases.size(), 0, false);
6583
6584         m_out.appendTo(slowBlock, lastNext);
6585         switchStringSlow(data, string);
6586     }
6587
6588     // The code for string switching is based closely on the same code in the DFG backend. While it
6589     // would be nice to reduce the amount of similar-looking code, it seems like this is one of
6590     // those algorithms where factoring out the common bits would result in more code than just
6591     // duplicating.
6592
6593     struct StringSwitchCase {
6594         StringSwitchCase() { }
6595
6596         StringSwitchCase(StringImpl* string, LBasicBlock target)
6597             : string(string)
6598             , target(target)
6599         {
6600         }
6601
6602         bool operator<(const StringSwitchCase& other) const
6603         {
6604             return stringLessThan(*string, *other.string);
6605         }
6606
6607         StringImpl* string;
6608         LBasicBlock target;
6609     };
6610
6611     struct CharacterCase {
6612         CharacterCase()
6613             : character(0)
6614             , begin(0)
6615             , end(0)
6616         {
6617         }
6618
6619         CharacterCase(LChar character, unsigned begin, unsigned end)
6620             : character(character)
6621             , begin(begin)
6622             , end(end)
6623         {
6624         }
6625
6626         bool operator<(const CharacterCase& other) const
6627         {
6628             return character < other.character;
6629         }
6630
6631         LChar character;
6632         unsigned begin;
6633         unsigned end;
6634     };
6635
6636     void switchStringRecurse(
6637         SwitchData* data, LValue buffer, LValue length, const Vector<StringSwitchCase>& cases,
6638         unsigned numChecked, unsigned begin, unsigned end, unsigned alreadyCheckedLength,
6639         unsigned checkedExactLength)
6640     {
6641         LBasicBlock fallThrough = lowBlock(data->fallThrough.block);
6642
6643         if (begin == end) {
6644             m_out.jump(fallThrough);
6645             return;
6646         }
6647
6648         unsigned minLength = cases[begin].string->length();
6649         unsigned commonChars = minLength;
6650         bool allLengthsEqual = true;
6651         for (unsigned i = begin + 1; i < end; ++i) {
6652             unsigned myCommonChars = numChecked;
6653             unsigned limit = std::min(cases[begin].string->length(), cases[i].string->length());
6654             for (unsigned j = numChecked; j < limit; ++j) {
6655                 if (cases[begin].string->at(j) != cases[i].string->at(j))
6656                     break;
6657                 myCommonChars++;
6658             }
6659             commonChars = std::min(commonChars, myCommonChars);
6660             if (minLength != cases[i].string->length())
6661                 allLengthsEqual = false;
6662             minLength = std::min(minLength, cases[i].string->length());
6663         }
6664
6665         if (checkedExactLength) {
6666             DFG_ASSERT(m_graph, m_node, alreadyCheckedLength == minLength);
6667             DFG_ASSERT(m_graph, m_node, allLengthsEqual);
6668         }
6669
6670         DFG_ASSERT(m_graph, m_node, minLength >= commonChars);
6671
6672         if (!allLengthsEqual && alreadyCheckedLength < minLength)
6673             m_out.check(m_out.below(length, m_out.constInt32(minLength)), unsure(fallThrough));
6674         if (allLengthsEqual && (alreadyCheckedLength < minLength || !checkedExactLength))
6675             m_out.check(m_out.notEqual(length, m_out.constInt32(minLength)), unsure(fallThrough));
6676
6677         for (unsigned i = numChecked; i < commonChars; ++i) {
6678             m_out.check(
6679                 m_out.notEqual(
6680                     m_out.load8(buffer, m_heaps.characters8[i]),
6681                     m_out.constInt8(cases[begin].string->at(i))),
6682                 unsure(fallThrough));
6683         }
6684
6685         if (minLength == commonChars) {
6686             // This is the case where one of the cases is a prefix of all of the other cases.
6687             // We've already checked that the input string is a prefix of all of the cases,
6688             // so we just check length to jump to that case.
6689
6690             DFG_ASSERT(m_graph, m_node, cases[begin].string->length() == commonChars);
6691             for (unsigned i = begin + 1; i < end; ++i)
6692                 DFG_ASSERT(m_graph, m_node, cases[i].string->length() > commonChars);
6693
6694             if (allLengthsEqual) {
6695                 DFG_ASSERT(m_graph, m_node, end == begin + 1);
6696                 m_out.jump(cases[begin].target);
6697                 return;
6698             }
6699
6700             m_out.check(
6701                 m_out.equal(length, m_out.constInt32(commonChars)),
6702                 unsure(cases[begin].target));
6703
6704             // We've checked if the length is >= minLength, and then we checked if the length is
6705             // == commonChars. We get to this point if it is >= minLength but not == commonChars.
6706             // Hence we know that it now must be > minLength, i.e. that it's >= minLength + 1.
6707             switchStringRecurse(
6708                 data, buffer, length, cases, commonChars, begin + 1, end, minLength + 1, false);
6709             return;
6710         }
6711
6712         // At this point we know that the string is longer than commonChars, and we've only verified
6713         // commonChars. Use a binary switch on the next unchecked character, i.e.
6714         // string[commonChars].
6715
6716         DFG_ASSERT(m_graph, m_node, end >= begin + 2);
6717
6718         LValue uncheckedChar = m_out.load8(buffer, m_heaps.characters8[commonChars]);
6719
6720         Vector<CharacterCase> characterCases;
6721         CharacterCase currentCase(cases[begin].string->at(commonChars), begin, begin + 1);
6722         for (unsigned i = begin + 1; i < end; ++i) {
6723             LChar currentChar = cases[i].string->at(commonChars);
6724             if (currentChar != currentCase.character) {
6725                 currentCase.end = i;
6726                 characterCases.append(currentCase);
6727                 currentCase = CharacterCase(currentChar, i, i + 1);
6728             } else
6729                 currentCase.end = i + 1;
6730         }
6731         characterCases.append(currentCase);
6732
6733         Vector<LBasicBlock> characterBlocks;
6734         for (CharacterCase& myCase : characterCases)
6735             characterBlocks.append(FTL_NEW_BLOCK(m_out, ("Switch/SwitchString case for ", myCase.character, " at index ", commonChars)));
6736
6737         Vector<SwitchCase> switchCases;
6738         for (unsigned i = 0; i < characterCases.size(); ++i) {
6739             if (i)
6740                 DFG_ASSERT(m_graph, m_node, characterCases[i - 1].character < characterCases[i].character);
6741             switchCases.append(SwitchCase(
6742                 m_out.constInt8(characterCases[i].character), characterBlocks[i], Weight()));
6743         }
6744         m_out.switchInstruction(uncheckedChar, switchCases, fallThrough, Weight());
6745
6746         LBasicBlock lastNext = m_out.m_nextBlock;
6747         characterBlocks.append(lastNext); // Makes it convenient to set nextBlock.
6748         for (unsigned i = 0; i < characterCases.size(); ++i) {
6749             m_out.appendTo(characterBlocks[i], characterBlocks[i + 1]);
6750             switchStringRecurse(
6751                 data, buffer, length, cases, commonChars + 1,
6752                 characterCases[i].begin, characterCases[i].end, minLength, allLengthsEqual);
6753         }
6754
6755         DFG_ASSERT(m_graph, m_node, m_out.m_nextBlock == lastNext);
6756     }
6757
6758     void switchStringSlow(SwitchData* data, LValue string)
6759     {
6760         // FIXME: We ought to be able to use computed gotos here. We would save the labels of the
6761         // blocks we want to jump to, and then request their addresses after compilation completes.
6762         // https://bugs.webkit.org/show_bug.cgi?id=144369
6763
6764         LValue branchOffset = vmCall(
6765             m_out.operation(operationSwitchStringAndGetBranchOffset),
6766             m_callFrame, m_out.constIntPtr(data->switchTableIndex), string);
6767
6768         StringJumpTable& table = codeBlock()->stringSwitchJumpTable(data->switchTableIndex);
6769
6770         Vector<SwitchCase> cases;
6771         std::unordered_set<int32_t> alreadyHandled; // These may be negative, or zero, or probably other stuff, too. We don't want to mess with HashSet's corner cases and we don't really care about throughput here.
6772         for (unsigned i = 0; i < data->cases.size(); ++i) {
6773             // FIXME: The fact that we're using the bytecode's switch table means that the
6774             // following DFG IR transformation would be invalid.
6775             //
6776             // Original code:
6777             //     switch (v) {
6778             //     case "foo":
6779             //     case "bar":
6780             //         things();
6781             //         break;
6782             //     default:
6783             //         break;
6784             //     }
6785             //
6786             // New code:
6787             //     switch (v) {
6788             //     case "foo":
6789             //         instrumentFoo();
6790             //         goto _things;
6791             //     case "bar":
6792             //         instrumentBar();
6793             //     _things:
6794             //         things();
6795             //         break;
6796             //     default:
6797             //         break;
6798             //     }
6799             //
6800             // Luckily, we don't currently do any such transformation. But it's kind of silly that
6801             // this is an issue.
6802             // https://bugs.webkit.org/show_bug.cgi?id=144635
6803
6804             DFG::SwitchCase myCase = data->cases[i];
6805             StringJumpTable::StringOffsetTable::iterator iter =
6806                 table.offsetTable.find(myCase.value.stringImpl());
6807             DFG_ASSERT(m_graph, m_node, iter != table.offsetTable.end());
6808
6809             if (!alreadyHandled.insert(iter->value.branchOffset).second)
6810                 continue;
6811
6812             cases.append(SwitchCase(
6813                 m_out.constInt32(iter->value.branchOffset),
6814                 lowBlock(myCase.target.block), Weight(myCase.target.count)));
6815         }
6816
6817         m_out.switchInstruction(
6818             branchOffset, cases, lowBlock(data->fallThrough.block),
6819             Weight(data->fallThrough.count));
6820     }
6821
6822     // Calls the functor at the point of code generation where we know what the result type is.
6823     // You can emit whatever code you like at that point. Expects you to terminate the basic block.
6824     // When buildTypeOf() returns, it will have terminated all basic blocks that it created. So, if
6825     // you aren't using this as the terminator of a high-level block, you should create your own
6826     // contination and set it as the nextBlock (m_out.insertNewBlocksBefore(continuation)) before
6827     // calling this. For example:
6828     //
6829     // LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("My continuation"));
6830     // LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
6831     // buildTypeOf(
6832     //     child, value,
6833     //     [&] (TypeofType type) {
6834     //          do things;
6835     //          m_out.jump(continuation);
6836     //     });
6837     // m_out.appendTo(continuation, lastNext);
6838     template<typename Functor>
6839     void buildTypeOf(Edge child, LValue value, const Functor& functor)
6840     {
6841         JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6842
6843         // Implements the following branching structure:
6844         //
6845         // if (is cell) {
6846         //     if (is object) {
6847         //         if (is function) {
6848         //             return function;
6849         //         } else if (doesn't have call trap and doesn't masquerade as undefined) {
6850         //             return object
6851         //         } else {
6852         //             return slowPath();
6853         //         }
6854         //     } else if (is string) {
6855         //         return string
6856         //     } else {
6857         //         return symbol
6858         //     }
6859         // } else if (is number) {
6860         //     return number
6861         // } else if (is null) {
6862         //     return object
6863         // } else if (is boolean) {
6864         //     return boolean
6865         // } else {
6866         //     return undefined
6867         // }
6868
6869         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf cell case"));
6870         LBasicBlock objectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf object case"));
6871         LBasicBlock functionCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf function case"));
6872         LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not function case"));
6873         LBasicBlock reallyObjectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf really object case"));
6874         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("buildTypeOf slow path"));
6875         LBasicBlock unreachable = FTL_NEW_BLOCK(m_out, ("buildTypeOf unreachable"));
6876         LBasicBlock notObjectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not object case"));
6877         LBasicBlock stringCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf string case"));
6878         LBasicBlock symbolCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf symbol case"));
6879         LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not cell case"));
6880         LBasicBlock numberCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf number case"));
6881         LBasicBlock notNumberCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not number case"));
6882         LBasicBlock notNullCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not null case"));
6883         LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf boolean case"));
6884         LBasicBlock undefinedCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf undefined case"));
6885
6886         m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase));
6887
6888         LBasicBlock lastNext = m_out.appendTo(cellCase, objectCase);
6889         m_out.branch(isObject(value, provenType(child)), unsure(objectCase), unsure(notObjectCase));
6890
6891         m_out.appendTo(objectCase, functionCase);
6892         m_out.branch(
6893             isFunction(value, provenType(child) & SpecObject),
6894             unsure(functionCase), unsure(notFunctionCase));
6895
6896         m_out.appendTo(functionCase, notFunctionCase);
6897         functor(TypeofType::Function);
6898
6899         m_out.appendTo(notFunctionCase, reallyObjectCase);
6900         m_out.branch(
6901             isExoticForTypeof(value, provenType(child) & (SpecObject - SpecFunction)),
6902             rarely(slowPath), usually(reallyObjectCase));
6903
6904         m_out.appendTo(reallyObjectCase, slowPath);
6905         functor(TypeofType::Object);
6906
6907         m_out.appendTo(slowPath, unreachable);
6908         LValue result = vmCall(
6909             m_out.operation(operationTypeOfObjectAsTypeofType), m_callFrame,
6910             weakPointer(globalObject), value);
6911         Vector<SwitchCase, 3> cases;
6912         cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Undefined)), undefinedCase));
6913         cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Object)), reallyObjectCase));
6914         cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Function)), functionCase));
6915         m_out.switchInstruction(result, cases, unreachable, Weight());
6916
6917         m_out.appendTo(unreachable, notObjectCase);
6918         m_out.unreachable();
6919
6920         m_out.appendTo(notObjectCase, stringCase);
6921         m_out.branch(
6922             isString(value, provenType(child) & (SpecCell - SpecObject)),
6923             unsure(stringCase), unsure(symbolCase));
6924
6925         m_out.appendTo(stringCase, symbolCase);
6926         functor(TypeofType::String);
6927
6928         m_out.appendTo(symbolCase, notCellCase);
6929         functor(TypeofType::Symbol);
6930
6931         m_out.appendTo(notCellCase, numberCase);
6932         m_out.branch(
6933             isNumber(value, provenType(child) & ~SpecCell),
6934             unsure(numberCase), unsure(notNumberCase));
6935
6936         m_out.appendTo(numberCase, notNumberCase);
6937         functor(TypeofType::Number);
6938
6939         m_out.appendTo(notNumberCase, notNullCase);
6940         LValue isNull;
6941         if (provenType(child) & SpecOther)
6942             isNull = m_out.equal(value, m_out.constInt64(ValueNull));
6943         else
6944             isNull = m_out.booleanFalse;
6945         m_out.branch(isNull, unsure(reallyObjectCase), unsure(notNullCase));
6946
6947         m_out.appendTo(notNullCase, booleanCase);
6948         m_out.branch(
6949             isBoolean(value, provenType(child) & ~(SpecCell | SpecFullNumber)),
6950             unsure(booleanCase), unsure(undefinedCase));
6951
6952         m_out.appendTo(booleanCase, undefinedCase);
6953         functor(TypeofType::Boolean);
6954
6955         m_out.appendTo(undefinedCase, lastNext);
6956         functor(TypeofType::Undefined);
6957     }
6958
6959     LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true)
6960     {
6961         LBasicBlock greatEnough = FTL_NEW_BLOCK(m_out, ("doubleToInt32 greatEnough"));
6962         LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("doubleToInt32 withinRange"));
6963         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("doubleToInt32 slowPath"));
6964         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("doubleToInt32 continuation"));
6965
6966         Vector<ValueFromBlock, 2> results;
6967
6968         m_out.branch(
6969             m_out.doubleGreaterThanOrEqual(doubleValue, m_out.constDouble(low)),
6970             unsure(greatEnough), unsure(slowPath));
6971
6972         LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange);
6973         m_out.branch(
6974             m_out.doubleLessThanOrEqual(doubleValue, m_out.constDouble(high)),
6975             unsure(withinRange), unsure(slowPath));
6976
6977         m_out.appendTo(withinRange, slowPath);
6978         LValue fastResult;
6979         if (isSigned)
6980             fastResult = m_out.fpToInt32(doubleValue);
6981         else
6982             fastResult = m_out.fpToUInt32(doubleValue);
6983         results.append(m_out.anchor(fastResult));
6984         m_out.jump(continuation);
6985
6986         m_out.appendTo(slowPath, continuation);
6987         results.append(m_out.anchor(m_out.call(m_out.operation(toInt32), doubleValue)));
6988         m_out.jump(continuation);
6989
6990         m_out.appendTo(continuation, lastNext);
6991         return m_out.phi(m_out.int32, results);
6992     }
6993
6994     LValue doubleToInt32(LValue doubleValue)
6995     {
6996         if (Output::hasSensibleDoubleToInt())
6997             return sensibleDoubleToInt32(doubleValue);
6998
6999         double limit = pow(2, 31) - 1;
7000         return doubleToInt32(doubleValue, -limit, limit);
7001     }
7002
7003     LValue sensibleDoubleToInt32(LValue doubleValue)
7004     {
7005         LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("sensible doubleToInt32 slow path"));
7006         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("sensible doubleToInt32 continuation"));
7007
7008         ValueFromBlock fastResult = m_out.anchor(
7009             m_out.sensibleDoubleToInt(doubleValue));
7010         m_out.branch(
7011             m_out.equal(fastResult.value(), m_out.constInt32(0x80000000)),
7012             rarely(slowPath), usually(continuation));
7013
7014         LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
7015         ValueFromBlock slowResult = m_out.anchor(
7016             m_out.call(m_out.operation(toInt32), doubleValue));
7017         m_out.jump(continuation);
7018
7019         m_out.appendTo(continuation, lastNext);
7020         return m_out.phi(m_out.int32, fastResult, slowResult);
7021     }
7022
7023     void speculate(
7024         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
7025     {
7026         appendOSRExit(kind, lowValue, highValue, failCondition);
7027     }
7028
7029     void terminate(ExitKind kind)
7030     {
7031         speculate(kind, noValue(), nullptr, m_out.booleanTrue);
7032         didAlreadyTerminate();
7033     }
7034
7035     void didAlreadyTerminate()
7036     {
7037         m_state.setIsValid(false);
7038     }
7039
7040     void typeCheck(
7041         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
7042         LValue failCondition)
7043     {
7044         appendTypeCheck(lowValue, highValue, typesPassedThrough, failCondition);
7045     }
7046
7047     void appendTypeCheck(
7048         FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
7049         LValue failCondition)
7050     {
7051         if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
7052             return;
7053         ASSERT(mayHaveTypeCheck(highValue.useKind()));
7054         appendOSRExit(BadType, lowValue, highValue.node(), failCondition);
7055         m_interpreter.filter(highValue, typesPassedThrough);
7056     }
7057
7058     LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7059     {
7060         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
7061
7062         if (edge->hasConstant()) {
7063             JSValue value = edge->asJSValue();
7064             if (!value.isInt32()) {
7065                 terminate(Uncountable);
7066                 return m_out.int32Zero;
7067             }
7068             return m_out.constInt32(value.asInt32());
7069         }
7070
7071         LoweredNodeValue value = m_int32Values.get(edge.node());
7072         if (isValid(value))
7073             return value.value();
7074
7075         value = m_strictInt52Values.get(edge.node());
7076         if (isValid(value))
7077             return strictInt52ToInt32(edge, value.value());
7078
7079         value = m_int52Values.get(edge.node());
7080         if (isValid(value))
7081             return strictInt52ToInt32(edge, int52ToStrictInt52(value.value()));
7082
7083         value = m_jsValueValues.get(edge.node());
7084         if (isValid(value)) {
7085             LValue boxedResult = value.value();
7086             FTL_TYPE_CHECK(
7087                 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
7088             LValue result = unboxInt32(boxedResult);
7089             setInt32(edge.node(), result);
7090             return result;
7091         }
7092
7093         DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecInt32));
7094         terminate(Uncountable);
7095         return m_out.int32Zero;
7096     }
7097
7098     enum Int52Kind { StrictInt52, Int52 };
7099     LValue lowInt52(Edge edge, Int52Kind kind)
7100     {
7101         DFG_ASSERT(m_graph, m_node, edge.useKind() == Int52RepUse);
7102
7103         LoweredNodeValue value;
7104
7105         switch (kind) {
7106         case Int52:
7107             value = m_int52Values.get(edge.node());
7108             if (isValid(value))
7109                 return value.value();
7110
7111             value = m_strictInt52Values.get(edge.node());
7112             if (isValid(value))
7113                 return strictInt52ToInt52(value.value());
7114             break;
7115
7116         case StrictInt52:
7117             value = m_strictInt52Values.get(edge.node());
7118             if (isValid(value))
7119                 return value.value();
7120
7121             value = m_int52Values.get(edge.node());
7122             if (isValid(value))
7123                 return int52ToStrictInt52(value.value());
7124             break;
7125         }
7126
7127         DFG_ASSERT(m_graph, m_node, !provenType(edge));
7128         terminate(Uncountable);
7129         return m_out.int64Zero;
7130     }
7131
7132     LValue lowInt52(Edge edge)
7133     {
7134         return lowInt52(edge, Int52);
7135     }
7136
7137     LValue lowStrictInt52(Edge edge)
7138     {
7139         return lowInt52(edge, StrictInt52);
7140     }
7141
7142     bool betterUseStrictInt52(Node* node)
7143     {
7144         return !isValid(m_int52Values.get(node));
7145     }
7146     bool betterUseStrictInt52(Edge edge)
7147     {
7148         return betterUseStrictInt52(edge.node());
7149     }
7150     template<typename T>
7151     Int52Kind bestInt52Kind(T node)
7152     {
7153         return betterUseStrictInt52(node) ? StrictInt52 : Int52;
7154     }
7155     Int52Kind opposite(Int52Kind kind)
7156     {
7157         switch (kind) {
7158         case Int52:
7159             return StrictInt52;
7160         case StrictInt52:
7161             return Int52;
7162         }
7163         DFG_CRASH(m_graph, m_node, "Bad use kind");
7164         return Int52;
7165     }
7166
7167     LValue lowWhicheverInt52(Edge edge, Int52Kind& kind)
7168     {
7169         kind = bestInt52Kind(edge);
7170         return lowInt52(edge, kind);
7171     }
7172
7173     LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7174     {
7175         DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || DFG::isCell(edge.useKind()));
7176
7177         if (edge->op() == JSConstant) {
7178             JSValue value = edge->asJSValue();
7179             if (!value.isCell()) {
7180                 terminate(Uncountable);
7181                 return m_out.intPtrZero;
7182             }
7183             return m_out.constIntPtr(value.asCell());
7184         }
7185
7186         LoweredNodeValue value = m_jsValueValues.get(edge.node());
7187         if (isValid(value)) {
7188             LValue uncheckedValue = value.value();
7189             FTL_TYPE_CHECK(
7190                 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
7191             return uncheckedValue;
7192         }
7193
7194         DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecCell));
7195         terminate(Uncountable);
7196         return m_out.intPtrZero;
7197     }
7198
7199     LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7200     {
7201         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
7202
7203         LValue result = lowCell(edge, mode);
7204         speculateObject(edge, result);
7205         return result;
7206     }
7207
7208     LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7209     {
7210         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse || edge.useKind() == StringIdentUse);
7211
7212         LValue result = lowCell(edge, mode);
7213         speculateString(edge, result);
7214         return result;
7215     }
7216
7217     LValue lowStringIdent(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7218     {
7219         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringIdentUse);
7220
7221         LValue string = lowString(edge, mode);
7222         LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
7223         speculateStringIdent(edge, string, stringImpl);
7224         return stringImpl;
7225     }
7226
7227     LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7228     {
7229         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
7230
7231         LValue result = lowCell(edge, mode);
7232         speculateNonNullObject(edge, result);
7233         return result;
7234     }
7235
7236     LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7237     {
7238         ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
7239
7240         if (edge->hasConstant()) {
7241             JSValue value = edge->asJSValue();
7242             if (!value.isBoolean()) {
7243                 terminate(Uncountable);
7244                 return m_out.booleanFalse;
7245             }
7246             return m_out.constBool(value.asBoolean());
7247         }
7248
7249         LoweredNodeValue value = m_booleanValues.get(edge.node());
7250         if (isValid(value))
7251             return value.value();
7252
7253         value = m_jsValueValues.get(edge.node());
7254         if (isValid(value)) {
7255             LValue unboxedResult = value.value();
7256             FTL_TYPE_CHECK(
7257                 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
7258             LValue result = unboxBoolean(unboxedResult);
7259             setBoolean(edge.node(), result);
7260             return result;
7261         }
7262
7263         DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecBoolean));
7264         terminate(Uncountable);
7265         return m_out.booleanFalse;
7266     }
7267
7268     LValue lowDouble(Edge edge)
7269     {
7270         DFG_ASSERT(m_graph, m_node, isDouble(edge.useKind()));
7271
7272         LoweredNodeValue value = m_doubleValues.get(edge.node());
7273         if (isValid(value))
7274             return value.value();
7275         DFG_ASSERT(m_graph, m_node, !provenType(edge));
7276         terminate(Uncountable);
7277         return m_out.doubleZero;
7278     }
7279
7280     LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7281     {
7282         DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
7283         DFG_ASSERT(m_graph, m_node, !isDouble(edge.useKind()));
7284         DFG_ASSERT(m_graph, m_node, edge.useKind() != Int52RepUse);
7285
7286         if (edge->hasConstant())
7287             return m_out.constInt64(JSValue::encode(edge->asJSValue()));
7288
7289         LoweredNodeValue value = m_jsValueValues.get(edge.node());
7290         if (isValid(value))
7291             return value.value();
7292
7293         value = m_int32Values.get(edge.node());
7294         if (isValid(value)) {
7295             LValue result = boxInt32(value.value());
7296             setJSValue(edge.node(), result);
7297             return result;
7298         }
7299
7300         value = m_booleanValues.get(edge.node());
7301         if (isValid(value)) {
7302             LValue result = boxBoolean(value.value());
7303             setJSValue(edge.node(), result);
7304             return result;
7305         }
7306
7307         DFG_CRASH(m_graph, m_node, "Value not defined");
7308         return 0;
7309     }
7310
7311     LValue lowStorage(Edge edge)
7312     {
7313         LoweredNodeValue value = m_storageValues.get(edge.node());
7314         if (isValid(value))
7315             return value.value();
7316
7317         LValue result = lowCell(edge);
7318         setStorage(edge.node(), result);
7319         return result;
7320     }
7321
7322     LValue strictInt52ToInt32(Edge edge, LValue value)
7323     {
7324         LValue result = m_out.castToInt32(value);
7325         FTL_TYPE_CHECK(
7326             noValue(), edge, SpecInt32,
7327             m_out.notEqual(m_out.signExt(result, m_out.int64), value));
7328         setInt32(edge.node(), result);
7329         return result;
7330     }
7331
7332     LValue strictInt52ToDouble(LValue value)
7333     {
7334         return m_out.intToDouble(value);
7335     }
7336
7337     LValue strictInt52ToJSValue(LValue value)
7338     {
7339         LBasicBlock isInt32 = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isInt32 case"));
7340         LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isDouble case"));
7341         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue continuation"));
7342
7343         Vector<ValueFromBlock, 2> results;
7344
7345         LValue int32Value = m_out.castToInt32(value);
7346         m_out.branch(
7347             m_out.equal(m_out.signExt(int32Value, m_out.int64), value),
7348             unsure(isInt32), unsure(isDouble));
7349
7350         LBasicBlock lastNext = m_out.appendTo(isInt32, isDouble);
7351
7352         results.append(m_out.anchor(boxInt32(int32Value)));
7353         m_out.jump(continuation);
7354
7355         m_out.appendTo(isDouble, continuation);
7356
7357         results.append(m_out.anchor(boxDouble(m_out.intToDouble(value))));
7358         m_out.jump(continuation);
7359
7360         m_out.appendTo(continuation, lastNext);
7361         return m_out.phi(m_out.int64, results);
7362     }
7363
7364     LValue strictInt52ToInt52(LValue value)
7365     {
7366         return m_out.shl(value, m_out.constInt64(JSValue::int52ShiftAmount));
7367     }
7368
7369     LValue int52ToStrictInt52(LValue value)
7370     {
7371         return m_out.aShr(value, m_out.constInt64(JSValue::int52ShiftAmount));
7372     }
7373
7374     LValue isInt32(LValue jsValue, SpeculatedType type = SpecFullTop)
7375     {
7376         if (LValue proven = isProvenValue(type, SpecInt32))
7377             return proven;
7378         return m_out.aboveOrEqual(jsValue, m_tagTypeNumber);
7379     }
7380     LValue isNotInt32(LValue jsValue, SpeculatedType type = SpecFullTop)
7381     {
7382         if (LValue proven = isProvenValue(type, ~SpecInt32))
7383             return proven;
7384         return m_out.below(jsValue, m_tagTypeNumber);
7385     }
7386     LValue unboxInt32(LValue jsValue)
7387     {
7388         return m_out.castToInt32(jsValue);
7389     }
7390     LValue boxInt32(LValue value)
7391     {
7392         return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
7393     }
7394
7395     LValue isCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop)
7396     {
7397         if (LValue proven = isProvenValue(type, SpecCell | SpecMisc))
7398             return proven;
7399         return m_out.testIsZero64(jsValue, m_tagTypeNumber);
7400     }
7401     LValue isNotCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop)
7402     {
7403         if (LValue proven = isProvenValue(type, ~(SpecCell | SpecMisc)))
7404             return proven;
7405         return m_out.testNonZero64(jsValue, m_tagTypeNumber);
7406     }
7407
7408     LValue unboxDouble(LValue jsValue)
7409     {
7410         return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
7411     }
7412     LValue boxDouble(LValue doubleValue)
7413     {
7414         return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
7415     }
7416
7417     LValue jsValueToStrictInt52(Edge edge, LValue boxedValue)
7418     {
7419         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing int case"));
7420         LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing double case"));
7421         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing continuation"));
7422
7423         LValue isNotInt32;
7424         if (!m_interpreter.needsTypeCheck(edge, SpecInt32))
7425             isNotInt32 = m_out.booleanFalse;
7426         else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32))
7427             isNotInt32 = m_out.booleanTrue;
7428         else
7429             isNotInt32 = this->isNotInt32(boxedValue);
7430         m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase));
7431
7432         LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
7433
7434         ValueFromBlock intToInt52 = m_out.anchor(
7435             m_out.signExt(unboxInt32(boxedValue), m_out.int64));
7436         m_out.jump(continuation);
7437
7438         m_out.appendTo(doubleCase, continuation);
7439
7440         LValue possibleResult = m_out.call(
7441             m_out.operation(operationConvertBoxedDoubleToInt52), boxedValue);
7442         FTL_TYPE_CHECK(
7443             jsValueValue(boxedValue), edge, SpecInt32 | SpecInt52AsDouble,
7444             m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
7445
7446         ValueFromBlock doubleToInt52 = m_out.anchor(possibleResult);
7447         m_out.jump(continuation);
7448
7449         m_out.appendTo(continuation, lastNext);
7450
7451         return m_out.phi(m_out.int64, intToInt52, doubleToInt52);
7452     }
7453
7454     LValue doubleToStrictInt52(Edge edge, LValue value)
7455     {
7456         LValue possibleResult = m_out.call(
7457             m_out.operation(operationConvertDoubleToInt52), value);
7458         FTL_TYPE_CHECK(
7459             doubleValue(value), edge, SpecInt52AsDouble,
7460             m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
7461
7462         return possibleResult;
7463     }
7464
7465     LValue convertDoubleToInt32(LValue value, bool shouldCheckNegativeZero)
7466     {
7467         LValue integerValue = m_out.fpToInt32(value);
7468         LValue integerValueConvertedToDouble = m_out.intToDouble(integerValue);
7469         LValue valueNotConvertibleToInteger = m_out.doubleNotEqualOrUnordered(value, integerValueConvertedToDouble);
7470         speculate(Overflow, FormattedValue(ValueFormatDouble, value), m_node, valueNotConvertibleToInteger);
7471
7472         if (shouldCheckNegativeZero) {
7473             LBasicBlock valueIsZero = FTL_NEW_BLOCK(m_out, ("ConvertDoubleToInt32 on zero"));
7474             LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ConvertDoubleToInt32 continuation"));
7475             m_out.branch(m_out.isZero32(integerValue), unsure(valueIsZero), unsure(continuation));
7476
7477             LBasicBlock lastNext = m_out.appendTo(valueIsZero, continuation);
7478
7479             LValue doubleBitcastToInt64 = m_out.bitCast(value, m_out.int64);
7480             LValue signBitSet = m_out.lessThan(doubleBitcastToInt64, m_out.constInt64(0));
7481
7482             speculate(NegativeZero, FormattedValue(ValueFormatDouble, value), m_node, signBitSet);
7483             m_out.jump(continuation);
7484             m_out.appendTo(continuation, lastNext);
7485         }
7486         return integerValue;
7487     }
7488
7489     LValue isNumber(LValue jsValue, SpeculatedType type = SpecFullTop)
7490     {
7491         if (LValue proven = isProvenValue(type, SpecFullNumber))
7492             return proven;
7493         return isNotCellOrMisc(jsValue);
7494     }
7495     LValue isNotNumber(LValue jsValue, SpeculatedType type = SpecFullTop)
7496     {
7497         if (LValue proven = isProvenValue(type, ~SpecFullNumber))
7498             return proven;
7499         return isCellOrMisc(jsValue);
7500     }
7501
7502     LValue isNotCell(LValue jsValue, SpeculatedType type = SpecFullTop)
7503     {
7504         if (LValue proven = isProvenValue(type, ~SpecCell))
7505             return proven;
7506         return m_out.testNonZero64(jsValue, m_tagMask);
7507     }
7508
7509     LValue isCell(LValue jsValue, SpeculatedType type = SpecFullTop)
7510     {
7511         if (LValue proven = isProvenValue(type, SpecCell))
7512             return proven;
7513         return m_out.testIsZero64(jsValue, m_tagMask);
7514     }
7515
7516     LValue isNotMisc(LValue value, SpeculatedType type = SpecFullTop)
7517     {
7518         if (LValue proven = isProvenValue(type, ~SpecMisc))
7519             return proven;
7520         return m_out.above(value, m_out.constInt64(TagBitTypeOther | TagBitBool | TagBitUndefined));
7521     }
7522
7523     LValue isMisc(LValue value, SpeculatedType type = SpecFullTop)
7524     {
7525         if (LValue proven = isProvenValue(type, SpecMisc))
7526             return proven;
7527         return m_out.bitNot(isNotMisc(value));
7528     }
7529
7530     LValue isNotBoolean(LValue jsValue, SpeculatedType type = SpecFullTop)
7531     {
7532         if (LValue proven = isProvenValue(type, ~SpecBoolean))
7533             return proven;
7534         return m_out.testNonZero64(
7535             m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
7536             m_out.constInt64(~1));
7537     }
7538     LValue isBoolean(LValue jsValue, SpeculatedType type = SpecFullTop)
7539     {
7540         if (LValue proven = isProvenValue(type, SpecBoolean))
7541             return proven;
7542         return m_out.bitNot(isNotBoolean(jsValue));
7543     }
7544     LValue unboxBoolean(LValue jsValue)
7545     {
7546         // We want to use a cast that guarantees that LLVM knows that even the integer
7547         // value is just 0 or 1. But for now we do it the dumb way.
7548         return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
7549     }
7550     LValue boxBoolean(LValue value)
7551     {
7552         return m_out.select(
7553             value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
7554     }
7555
7556     LValue isNotOther(LValue value, SpeculatedType type = SpecFullTop)
7557     {
7558         if (LValue proven = isProvenValue(type, ~SpecOther))
7559             return proven;
7560         return m_out.notEqual(
7561             m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
7562             m_out.constInt64(ValueNull));
7563     }
7564     LValue isOther(LValue value, SpeculatedType type = SpecFullTop)
7565     {
7566         if (LValue proven = isProvenValue(type, SpecOther))
7567             return proven;
7568         return m_out.equal(
7569             m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
7570             m_out.constInt64(ValueNull));
7571     }
7572
7573     LValue isProvenValue(SpeculatedType provenType, SpeculatedType wantedType)
7574     {
7575         if (!(provenType & ~wantedType))
7576             return m_out.booleanTrue;
7577         if (!(provenType & wantedType))
7578             return m_out.booleanFalse;
7579         return nullptr;
7580     }
7581
7582     void speculate(Edge edge)
7583     {
7584         switch (edge.useKind()) {
7585         case UntypedUse:
7586             break;
7587         case KnownInt32Use:
7588         case KnownStringUse:
7589         case DoubleRepUse:
7590         case Int52RepUse:
7591             ASSERT(!m_interpreter.needsTypeCheck(edge));
7592             break;
7593         case Int32Use:
7594             speculateInt32(edge);
7595             break;
7596         case CellUse:
7597             speculateCell(edge);
7598             break;
7599         case KnownCellUse:
7600             ASSERT(!m_interpreter.needsTypeCheck(edge));
7601             break;
7602         case MachineIntUse:
7603             speculateMachineInt(edge);
7604             break;
7605         case ObjectUse:
7606             speculateObject(edge);
7607             break;
7608         case FunctionUse:
7609             speculateFunction(edge);
7610             break;
7611         case ObjectOrOtherUse:
7612             speculateObjectOrOther(edge);
7613             break;
7614         case FinalObjectUse:
7615             speculateFinalObject(edge);
7616             break;
7617         case StringUse:
7618             speculateString(edge);
7619             break;
7620         case StringIdentUse:
7621             speculateStringIdent(edge);
7622             break;
7623         case StringObjectUse:
7624             speculateStringObject(edge);
7625             break;
7626         case StringOrStringObjectUse:
7627             speculateStringOrStringObject(edge);
7628             break;
7629         case NumberUse:
7630             speculateNumber(edge);
7631             break;
7632         case RealNumberUse:
7633             speculateRealNumber(edge);
7634             break;
7635         case DoubleRepRealUse:
7636             speculateDoubleRepReal(edge);
7637             break;
7638         case DoubleRepMachineIntUse:
7639             speculateDoubleRepMachineInt(edge);
7640             break;
7641         case BooleanUse:
7642             speculateBoolean(edge);
7643             break;
7644         case NotStringVarUse:
7645             speculateNotStringVar(edge);
7646             break;
7647         case NotCellUse:
7648             speculateNotCell(edge);
7649             break;
7650         case OtherUse:
7651             speculateOther(edge);
7652             break;
7653         case MiscUse:
7654             speculateMisc(edge);
7655             break;
7656         default:
7657             DFG_CRASH(m_graph, m_node, "Unsupported speculation use kind");
7658         }
7659     }
7660
7661     void speculate(Node*, Edge edge)
7662     {
7663         speculate(edge);
7664     }
7665
7666     void speculateInt32(Edge edge)
7667     {
7668         lowInt32(edge);
7669     }
7670
7671     void speculateCell(Edge edge)
7672     {
7673         lowCell(edge);
7674     }
7675
7676     void speculateMachineInt(Edge edge)
7677     {
7678         if (!m_interpreter.needsTypeCheck(edge))
7679             return;
7680
7681         jsValueToStrictInt52(edge, lowJSValue(edge, ManualOperandSpeculation));
7682     }
7683
7684     LValue isObject(LValue cell, SpeculatedType type = SpecFullTop)
7685     {
7686         if (LValue proven = isProvenValue(type & SpecCell, SpecObject))
7687             return proven;
7688         return m_out.aboveOrEqual(
7689             m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7690             m_out.constInt8(ObjectType));
7691     }
7692
7693     LValue isNotObject(LValue cell, SpeculatedType type = SpecFullTop)
7694     {
7695         if (LValue proven = isProvenValue(type & SpecCell, ~SpecObject))
7696             return proven;
7697         return m_out.below(
7698             m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7699             m_out.constInt8(ObjectType));
7700     }
7701
7702     LValue isNotString(LValue cell, SpeculatedType type = SpecFullTop)
7703     {
7704         if (LValue proven = isProvenValue(type & SpecCell, ~SpecString))
7705             return proven;
7706         return m_out.notEqual(
7707             m_out.load32(cell, m_heaps.JSCell_structureID),
7708             m_out.constInt32(vm().stringStructure->id()));
7709     }
7710
7711     LValue isString(LValue cell, SpeculatedType type = SpecFullTop)
7712     {
7713         if (LValue proven = isProvenValue(type & SpecCell, SpecString))
7714             return proven;
7715         return m_out.equal(
7716             m_out.load32(cell, m_heaps.JSCell_structureID),
7717             m_out.constInt32(vm().stringStructure->id()));
7718     }
7719
7720     LValue isArrayType(LValue cell, ArrayMode arrayMode)
7721     {
7722         switch (arrayMode.type()) {
7723         case Array::Int32:
7724         case Array::Double:
7725         case Array::Contiguous: {
7726             LValue indexingType = m_out.load8(cell, m_heaps.JSCell_indexingType);
7727
7728             switch (arrayMode.arrayClass()) {
7729             case Array::OriginalArray:
7730                 DFG_CRASH(m_graph, m_node, "Unexpected original array");
7731                 return 0;
7732
7733             case Array::Array:
7734                 return m_out.equal(
7735                     m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
7736                     m_out.constInt8(IsArray | arrayMode.shapeMask()));
7737
7738             case Array::NonArray:
7739             case Array::OriginalNonArray:
7740                 return m_out.equal(
7741                     m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
7742                     m_out.constInt8(arrayMode.shapeMask()));
7743
7744             case Array::PossiblyArray:
7745                 return m_out.equal(
7746                     m_out.bitAnd(indexingType, m_out.constInt8(IndexingShapeMask)),
7747                     m_out.constInt8(arrayMode.shapeMask()));
7748             }
7749
7750             DFG_CRASH(m_graph, m_node, "Corrupt array class");
7751         }
7752
7753         case Array::DirectArguments:
7754             return m_out.equal(
7755                 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7756                 m_out.constInt8(DirectArgumentsType));
7757
7758         case Array::ScopedArguments:
7759             return m_out.equal(
7760                 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7761                 m_out.constInt8(ScopedArgumentsType));
7762
7763         default:
7764             return m_out.equal(
7765                 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7766                 m_out.constInt8(typeForTypedArrayType(arrayMode.typedArrayType())));
7767         }
7768     }
7769
7770     LValue isFunction(LValue cell, SpeculatedType type = SpecFullTop)
7771     {
7772         if (LValue proven = isProvenValue(type & SpecCell, SpecFunction))
7773             return proven;
7774         return isType(cell, JSFunctionType);
7775     }
7776     LValue isNotFunction(LValue cell, SpeculatedType type = SpecFullTop)
7777     {
7778         if (LValue proven = isProvenValue(type & SpecCell, ~SpecFunction))
7779             return proven;
7780         return isNotType(cell, JSFunctionType);
7781     }
7782
7783     LValue isExoticForTypeof(LValue cell, SpeculatedType type = SpecFullTop)
7784     {
7785         if (!(type & SpecObjectOther))
7786             return m_out.booleanFalse;
7787         return m_out.testNonZero8(
7788             m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
7789             m_out.constInt8(MasqueradesAsUndefined | TypeOfShouldCallGetCallData));
7790     }
7791
7792     LValue isType(LValue cell, JSType type)
7793     {
7794         return m_out.equal(
7795             m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7796             m_out.constInt8(type));
7797     }
7798
7799     LValue isNotType(LValue cell, JSType type)
7800     {
7801         return m_out.bitNot(isType(cell, type));
7802     }
7803
7804     void speculateObject(Edge edge, LValue cell)
7805     {
7806         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
7807     }
7808
7809     void speculateObject(Edge edge)
7810     {
7811         speculateObject(edge, lowCell(edge));
7812     }
7813
7814     void speculateFunction(Edge edge, LValue cell)
7815     {
7816         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecFunction, isNotFunction(cell));
7817     }
7818
7819     void speculateFunction(Edge edge)
7820     {
7821         speculateFunction(edge, lowCell(edge));
7822     }
7823
7824     void speculateObjectOrOther(Edge edge)
7825     {
7826         if (!m_interpreter.needsTypeCheck(edge))
7827             return;
7828
7829         LValue value = lowJSValue(edge, ManualOperandSpeculation);
7830
7831         LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther cell case"));
7832         LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther primitive case"));
7833         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther continuation"));
7834
7835         m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase));
7836
7837         LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
7838
7839         FTL_TYPE_CHECK(
7840             jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value));
7841
7842         m_out.jump(continuation);
7843
7844         m_out.appendTo(primitiveCase, continuation);
7845
7846         FTL_TYPE_CHECK(
7847             jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value));
7848
7849         m_out.jump(continuation);
7850
7851         m_out.appendTo(continuation, lastNext);
7852     }
7853
7854     void speculateFinalObject(Edge edge, LValue cell)
7855     {
7856         FTL_TYPE_CHECK(
7857             jsValueValue(cell), edge, SpecFinalObject, isNotType(cell, FinalObjectType));
7858     }
7859
7860     void speculateFinalObject(Edge edge)
7861     {
7862         speculateFinalObject(edge, lowCell(edge));
7863     }
7864
7865     void speculateString(Edge edge, LValue cell)
7866     {
7867         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString | ~SpecCell, isNotString(cell));
7868     }
7869
7870     void speculateString(Edge edge)
7871     {
7872         speculateString(edge, lowCell(edge));
7873     }
7874
7875     void speculateStringIdent(Edge edge, LValue string, LValue stringImpl)
7876     {
7877         if (!m_interpreter.needsTypeCheck(edge, SpecStringIdent | ~SpecString))
7878             return;
7879
7880         speculate(BadType, jsValueValue(string), edge.node(), m_out.isNull(stringImpl));
7881         speculate(
7882             BadType, jsValueValue(string), edge.node(),
7883             m_out.testIsZero32(
7884                 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
7885                 m_out.constInt32(StringImpl::flagIsAtomic())));
7886         m_interpreter.filter(edge, SpecStringIdent | ~SpecString);
7887     }
7888
7889     void speculateStringIdent(Edge edge)
7890     {
7891         lowStringIdent(edge);
7892     }
7893
7894     void speculateStringObject(Edge edge)
7895     {
7896         if (!m_interpreter.needsTypeCheck(edge, SpecStringObject))
7897             return;
7898
7899         speculateStringObjectForCell(edge, lowCell(edge));
7900         m_interpreter.filter(edge, SpecStringObject);
7901     }
7902
7903     void speculateStringOrStringObject(Edge edge)
7904     {
7905         if (!m_interpreter.needsTypeCheck(edge, SpecString | SpecStringObject))
7906             return;
7907
7908         LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject not string case"));
7909         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject continuation"));
7910
7911         LValue structureID = m_out.load32(lowCell(edge), m_heaps.JSCell_structureID);
7912         m_out.branch(
7913             m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())),
7914             unsure(continuation), unsure(notString));
7915
7916         LBasicBlock lastNext = m_out.appendTo(notString, continuation);
7917         speculateStringObjectForStructureID(edge, structureID);
7918         m_out.jump(continuation);
7919
7920         m_out.appendTo(continuation, lastNext);
7921
7922         m_interpreter.filter(edge, SpecString | SpecStringObject);
7923     }
7924
7925     void speculateStringObjectForCell(Edge edge, LValue cell)
7926     {
7927         speculateStringObjectForStructureID(edge, m_out.load32(cell, m_heaps.JSCell_structureID));
7928     }
7929
7930     void speculateStringObjectForStructureID(Edge edge, LValue structureID)
7931     {
7932         Structure* stringObjectStructure =
7933             m_graph.globalObjectFor(m_node->origin.semantic)->stringObjectStructure();
7934
7935         if (abstractStructure(edge).isSubsetOf(StructureSet(stringObjectStructure)))
7936             return;
7937
7938         speculate(
7939             NotStringObject, noValue(), 0,
7940             m_out.notEqual(structureID, weakStructureID(stringObjectStructure)));
7941     }
7942
7943     void speculateNonNullObject(Edge edge, LValue cell)
7944     {
7945         FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
7946         if (masqueradesAsUndefinedWatchpointIsStillValid())
7947             return;
7948
7949         speculate(
7950             BadType, jsValueValue(cell), edge.node(),
7951             m_out.testNonZero8(
7952                 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
7953                 m_out.constInt8(MasqueradesAsUndefined)));
7954     }
7955
7956     void speculateNumber(Edge edge)
7957     {
7958         LValue value = lowJSValue(edge, ManualOperandSpeculation);
7959         FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isNotNumber(value));
7960     }
7961
7962     void speculateRealNumber(Edge edge)
7963     {
7964         // Do an early return here because lowDouble() can create a lot of control flow.
7965         if (!m_interpreter.needsTypeCheck(edge))
7966             return;
7967
7968         LValue value = lowJSValue(edge, ManualOperandSpeculation);
7969         LValue doubleValue = unboxDouble(value);
7970
7971         LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("speculateRealNumber int case"));
7972         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateRealNumber continuation"));
7973
7974         m_out.branch(
7975             m_out.doubleEqual(doubleValue, doubleValue),
7976             usually(continuation), rarely(intCase));
7977
7978         LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
7979
7980         typeCheck(
7981             jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
7982             isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble));
7983         m_out.jump(continuation);
7984
7985         m_out.appendTo(continuation, lastNext);
7986     }
7987
7988     void speculateDoubleRepReal(Edge edge)
7989     {
7990         // Do an early return here because lowDouble() can create a lot of control flow.
7991         if (!m_interpreter.needsTypeCheck(edge))
7992             return;
7993
7994         LValue value = lowDouble(edge);
7995         FTL_TYPE_CHECK(
7996             doubleValue(value), edge, SpecDoubleReal,
7997             m_out.doubleNotEqualOrUnordered(value, value));
7998     }
7999
8000     void speculateDoubleRepMachineInt(Edge edge)
8001     {
8002         if (!m_interpreter.needsTypeCheck(edge))
8003             return;
8004
8005         doubleToStrictInt52(edge, lowDouble(edge));
8006     }
8007
8008     void speculateBoolean(Edge edge)
8009     {
8010         lowBoolean(edge);
8011     }
8012
8013     void speculateNotStringVar(Edge edge)
8014     {
8015         if (!m_interpreter.needsTypeCheck(edge, ~SpecStringVar))
8016             return;
8017
8018         LValue value = lowJSValue(edge, ManualOperandSpeculation);
8019
8020         LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is cell case"));
8021         LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is string case"));
8022         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar continuation"));
8023
8024         m_out.branch(isCell(value, provenType(edge)), unsure(isCellCase), unsure(continuation));
8025
8026         LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
8027         m_out.branch(isString(value, provenType(edge)), unsure(isStringCase), unsure(continuation));
8028
8029         m_out.appendTo(isStringCase, continuation);
8030         speculateStringIdent(edge, value, m_out.loadPtr(value, m_heaps.JSString_value));
8031         m_out.jump(continuation);
8032
8033         m_out.appendTo(continuation, lastNext);
8034     }
8035
8036     void speculateNotCell(Edge edge)
8037     {
8038         if (!m_interpreter.needsTypeCheck(edge))
8039             return;
8040
8041         LValue value = lowJSValue(edge, ManualOperandSpeculation);
8042         typeCheck(jsValueValue(value), edge, ~SpecCell, isCell(value));
8043     }
8044
8045     void speculateOther(Edge edge)
8046     {
8047         if (!m_interpreter.needsTypeCheck(edge))
8048             return;
8049
8050         LValue value = lowJSValue(edge, ManualOperandSpeculation);
8051         typeCheck(jsValueValue(value), edge, SpecOther, isNotOther(value));
8052     }
8053
8054     void speculateMisc(Edge edge)
8055     {
8056         if (!m_interpreter.needsTypeCheck(edge))
8057             return;
8058
8059         LValue value = lowJSValue(edge, ManualOperandSpeculation);
8060         typeCheck(jsValueValue(value), edge, SpecMisc, isNotMisc(value));
8061     }
8062
8063     bool masqueradesAsUndefinedWatchpointIsStillValid()
8064     {
8065         return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->origin.semantic);
8066     }
8067
8068     LValue loadMarkByte(LValue base)
8069     {
8070         return m_out.load8(base, m_heaps.JSCell_gcData);
8071     }
8072
8073     void emitStoreBarrier(LValue base)
8074     {
8075 #if ENABLE(GGC)
8076         LBasicBlock isMarkedAndNotRemembered = FTL_NEW_BLOCK(m_out, ("Store barrier is marked block"));
8077         LBasicBlock bufferHasSpace = FTL_NEW_BLOCK(m_out, ("Store barrier buffer has space"));
8078         LBasicBlock bufferIsFull = FTL_NEW_BLOCK(m_out, ("Store barrier buffer is full"));
8079         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation"));
8080
8081         // Check the mark byte.
8082         m_out.branch(
8083             m_out.notZero8(loadMarkByte(base)), usually(continuation), rarely(isMarkedAndNotRemembered));
8084
8085         // Append to the write barrier buffer.
8086         LBasicBlock lastNext = m_out.appendTo(isMarkedAndNotRemembered, bufferHasSpace);
8087         LValue currentBufferIndex = m_out.load32(m_out.absolute(vm().heap.writeBarrierBuffer().currentIndexAddress()));
8088         LValue bufferCapacity = m_out.constInt32(vm().heap.writeBarrierBuffer().capacity());
8089         m_out.branch(
8090             m_out.lessThan(currentBufferIndex, bufferCapacity),
8091             usually(bufferHasSpace), rarely(bufferIsFull));
8092
8093         // Buffer has space, store to it.
8094         m_out.appendTo(bufferHasSpace, bufferIsFull);
8095         LValue writeBarrierBufferBase = m_out.constIntPtr(vm().heap.writeBarrierBuffer().buffer());
8096         m_out.storePtr(base, m_out.baseIndex(m_heaps.WriteBarrierBuffer_bufferContents, writeBarrierBufferBase, m_out.zeroExtPtr(currentBufferIndex)));
8097         m_out.store32(m_out.add(currentBufferIndex, m_out.constInt32(1)), m_out.absolute(vm().heap.writeBarrierBuffer().currentIndexAddress()));
8098         m_out.jump(continuation);
8099
8100         // Buffer is out of space, flush it.
8101         m_out.appendTo(bufferIsFull, continuation);
8102         vmCallNoExceptions(m_out.operation(operationFlushWriteBarrierBuffer), m_callFrame, base);
8103         m_out.jump(continuation);
8104
8105         m_out.appendTo(continuation, lastNext);
8106 #else
8107         UNUSED_PARAM(base);
8108 #endif
8109     }
8110
8111     template<typename... Args>
8112     LValue vmCall(LValue function, Args... args)
8113     {
8114         callPreflight();
8115         LValue result = m_out.call(function, args...);
8116         callCheck();
8117         return result;
8118     }
8119
8120     template<typename... Args>
8121     LValue vmCallNoExceptions(LValue function, Args... args)
8122     {
8123         callPreflight();
8124         LValue result = m_out.call(function, args...);
8125         return result;
8126     }
8127
8128     void callPreflight(CodeOrigin codeOrigin)
8129     {
8130         m_out.store32(
8131             m_out.constInt32(
8132                 CallFrame::Location::encodeAsCodeOriginIndex(
8133                     m_ftlState.jitCode->common.addCodeOrigin(codeOrigin))),
8134             tagFor(JSStack::ArgumentCount));
8135     }
8136     void callPreflight()
8137     {
8138         callPreflight(m_node->origin.semantic);
8139     }
8140
8141     void callCheck()
8142     {
8143         if (Options::enableExceptionFuzz())
8144             m_out.call(m_out.operation(operationExceptionFuzz));
8145
8146         LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
8147
8148         LValue exception = m_out.load64(m_out.absolute(vm().addressOfException()));
8149
8150         m_out.branch(
8151             m_out.notZero64(exception), rarely(m_handleExceptions), usually(continuation));
8152
8153         m_out.appendTo(continuation);
8154     }
8155
8156     LBasicBlock lowBlock(BasicBlock* block)
8157     {
8158         return m_blocks.get(block);
8159     }
8160
8161     void appendOSRExit(
8162         ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
8163     {
8164         if (verboseCompilationEnabled()) {
8165             dataLog("    OSR exit #", m_ftlState.jitCode->osrExit.size(), " with availability: ", availabilityMap(), "\n");
8166             if (!m_availableRecoveries.isEmpty())
8167                 dataLog("        Available recoveries: ", listDump(m_availableRecoveries), "\n");
8168         }
8169
8170         if (doOSRExitFuzzing()) {
8171             LValue numberOfFuzzChecks = m_out.add(
8172                 m_out.load32(m_out.absolute(&g_numberOfOSRExitFuzzChecks)),
8173                 m_out.int32One);
8174
8175             m_out.store32(numberOfFuzzChecks, m_out.absolute(&g_numberOfOSRExitFuzzChecks));
8176
8177             if (unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter()) {
8178                 failCondition = m_out.bitOr(
8179                     failCondition,
8180                     m_out.aboveOrEqual(numberOfFuzzChecks, m_out.constInt32(atOrAfter)));
8181             }
8182             if (unsigned at = Options::fireOSRExitFuzzAt()) {
8183                 failCondition = m_out.bitOr(
8184                     failCondition,
8185                     m_out.equal(numberOfFuzzChecks, m_out.constInt32(at)));
8186             }
8187         }
8188
8189         ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.finalizer->osrExit.size());
8190
8191         m_ftlState.jitCode->osrExit.append(OSRExit(
8192             kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
8193             m_codeOriginForExitTarget, m_codeOriginForExitProfile,
8194             availabilityMap().m_locals.numberOfArguments(),
8195             availabilityMap().m_locals.numberOfLocals()));
8196         m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo());
8197
8198         OSRExit& exit = m_ftlState.jitCode->osrExit.last();
8199
8200         LBasicBlock lastNext = nullptr;
8201         LBasicBlock continuation = nullptr;
8202
8203         LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
8204         continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
8205
8206         m_out.branch(failCondition, rarely(failCase), usually(continuation));
8207
8208         lastNext = m_out.appendTo(failCase, continuation);
8209
8210         emitOSRExitCall(exit, lowValue);
8211
8212         m_out.unreachable();
8213
8214         m_out.appendTo(continuation, lastNext);
8215     }
8216
8217     void emitOSRExitCall(OSRExit& exit, FormattedValue lowValue)
8218     {
8219         ExitArgumentList arguments;
8220
8221         CodeOrigin codeOrigin = exit.m_codeOrigin;
8222
8223         buildExitArguments(exit, arguments, lowValue, codeOrigin);
8224
8225         callStackmap(exit, arguments);
8226     }
8227
8228     void buildExitArguments(
8229         OSRExit& exit, ExitArgumentList& arguments, FormattedValue lowValue,
8230         CodeOrigin codeOrigin)
8231     {
8232         if (!!lowValue)
8233             arguments.append(lowValue.value());
8234
8235         AvailabilityMap availabilityMap = this->availabilityMap();
8236         availabilityMap.pruneByLiveness(m_graph, codeOrigin);
8237
8238         HashMap<Node*, ExitTimeObjectMaterialization*> map;
8239         availabilityMap.forEachAvailability(
8240             [&] (Availability availability) {
8241                 if (!availability.shouldUseNode())
8242                     return;
8243
8244                 Node* node = availability.node();
8245                 if (!node->isPhantomAllocation())
8246                     return;
8247
8248                 auto result = map.add(node, nullptr);
8249                 if (result.isNewEntry) {
8250                     result.iterator->value =
8251                         exit.m_materializations.add(node->op(), node->origin.semantic);
8252                 }
8253             });
8254
8255         for (unsigned i = 0; i < exit.m_values.size(); ++i) {
8256             int operand = exit.m_values.operandForIndex(i);
8257
8258             Availability availability = availabilityMap.m_locals[i];
8259
8260             if (Options::validateFTLOSRExitLiveness()) {
8261                 DFG_ASSERT(
8262                     m_graph, m_node,
8263                     (!(availability.isDead() && m_graph.isLiveInBytecode(VirtualRegister(operand), codeOrigin))) || m_graph.m_plan.mode == FTLForOSREntryMode);
8264             }
8265
8266             exit.m_values[i] = exitValueForAvailability(arguments, map, availability);
8267         }
8268
8269         for (auto heapPair : availabilityMap.m_heap) {
8270             Node* node = heapPair.key.base();
8271             ExitTimeObjectMaterialization* materialization = map.get(node);
8272             materialization->add(
8273                 heapPair.key.descriptor(),
8274                 exitValueForAvailability(arguments, map, heapPair.value));
8275         }
8276
8277         if (verboseCompilationEnabled()) {
8278             dataLog("        Exit values: ", exit.m_values, "\n");
8279             if (!exit.m_materializations.isEmpty()) {
8280                 dataLog("        Materializations: \n");
8281                 for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
8282                     dataLog("            ", pointerDump(materialization), "\n");
8283             }
8284         }
8285     }
8286
8287     void callStackmap(OSRExit& exit, ExitArgumentList& arguments)
8288     {
8289         exit.m_stackmapID = m_stackmapIDs++;
8290         arguments.insert(0, m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
8291         arguments.insert(0, m_out.constInt64(exit.m_stackmapID));
8292
8293         m_out.call(m_out.stackmapIntrinsic(), arguments);
8294     }
8295
8296     ExitValue exitValueForAvailability(
8297         ExitArgumentList& arguments, const HashMap<Node*, ExitTimeObjectMaterialization*>& map,
8298         Availability availability)
8299     {
8300         FlushedAt flush = availability.flushedAt();
8301         switch (flush.format()) {
8302         case DeadFlush:
8303         case ConflictingFlush:
8304             if (availability.hasNode())
8305                 return exitValueForNode(arguments, map, availability.node());
8306
8307             // This means that the value is dead. It could be dead in bytecode or it could have
8308             // been killed by our DCE, which can sometimes kill things even if they were live in
8309             // bytecode.
8310             return ExitValue::dead();
8311
8312         case FlushedJSValue:
8313         case FlushedCell:
8314         case FlushedBoolean:
8315             return ExitValue::inJSStack(flush.virtualRegister());
8316
8317         case FlushedInt32:
8318             return ExitValue::inJSStackAsInt32(flush.virtualRegister());
8319
8320         case FlushedInt52:
8321             return ExitValue::inJSStackAsInt52(flush.virtualRegister());
8322
8323         case FlushedDouble:
8324             return ExitValue::inJSStackAsDouble(flush.virtualRegister());
8325         }
8326
8327         DFG_CRASH(m_graph, m_node, "Invalid flush format");
8328         return ExitValue::dead();
8329     }
8330
8331     ExitValue exitValueForNode(
8332         ExitArgumentList& arguments, const HashMap<Node*, ExitTimeObjectMaterialization*>& map,
8333         Node* node)
8334     {
8335         ASSERT(node->shouldGenerate());
8336         ASSERT(node->hasResult());
8337
8338         if (node) {
8339             switch (node->op()) {
8340             case BottomValue:
8341                 // This might arise in object materializations. I actually doubt that it would,
8342                 // but it seems worthwhile to be conservative.
8343                 return ExitValue::dead();
8344
8345             case JSConstant:
8346             case Int52Constant:
8347             case DoubleConstant:
8348                 return ExitValue::constant(node->asJSValue());
8349
8350             default:
8351                 if (node->isPhantomAllocation())
8352                     return ExitValue::materializeNewObject(map.get(node));
8353                 break;
8354             }
8355         }
8356
8357         for (unsigned i = 0; i < m_availableRecoveries.size(); ++i) {
8358             AvailableRecovery recovery = m_availableRecoveries[i];
8359             if (recovery.node() != node)
8360                 continue;
8361
8362             ExitValue result = ExitValue::recovery(
8363                 recovery.opcode(), arguments.size(), arguments.size() + 1,
8364                 recovery.format());
8365             arguments.append(recovery.left());
8366             arguments.append(recovery.right());
8367             return result;
8368         }
8369
8370         LoweredNodeValue value = m_int32Values.get(node);
8371         if (isValid(value))
8372             return exitArgument(arguments, ValueFormatInt32, value.value());
8373
8374         value = m_int52Values.get(node);
8375         if (isValid(value))
8376             return exitArgument(arguments, ValueFormatInt52, value.value());
8377
8378         value = m_strictInt52Values.get(node);
8379         if (isValid(value))
8380             return exitArgument(arguments, ValueFormatStrictInt52, value.value());
8381
8382         value = m_booleanValues.get(node);
8383         if (isValid(value)) {
8384             LValue valueToPass = m_out.zeroExt(value.value(), m_out.int32);
8385             return exitArgument(arguments, ValueFormatBoolean, valueToPass);
8386         }
8387
8388         value = m_jsValueValues.get(node);
8389         if (isValid(value))
8390             return exitArgument(arguments, ValueFormatJSValue, value.value());
8391
8392         value = m_doubleValues.get(node);
8393         if (isValid(value))
8394             return exitArgument(arguments, ValueFormatDouble, value.value());
8395
8396         DFG_CRASH(m_graph, m_node, toCString("Cannot find value for node: ", node).data());
8397         return ExitValue::dead();
8398     }
8399
8400     ExitValue exitArgument(ExitArgumentList& arguments, ValueFormat format, LValue value)
8401     {
8402         ExitValue result = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
8403         arguments.append(value);
8404         return result;
8405     }
8406
8407     bool doesKill(Edge edge)
8408     {
8409         if (edge.doesNotKill())
8410             return false;
8411
8412         if (edge->hasConstant())
8413             return false;
8414
8415         return true;
8416     }
8417
8418     void addAvailableRecovery(
8419         Node* node, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format)
8420     {
8421         m_availableRecoveries.append(AvailableRecovery(node, opcode, left, right, format));
8422     }
8423
8424     void addAvailableRecovery(
8425         Edge edge, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format)
8426     {
8427         addAvailableRecovery(edge.node(), opcode, left, right, format);
8428     }
8429
8430     void setInt32(Node* node, LValue value)
8431     {
8432         m_int32Values.set(node, LoweredNodeValue(value, m_highBlock));
8433     }
8434     void setInt52(Node* node, LValue value)
8435     {
8436         m_int52Values.set(node, LoweredNodeValue(value, m_highBlock));
8437     }
8438     void setStrictInt52(Node* node, LValue value)
8439     {
8440         m_strictInt52Values.set(node, LoweredNodeValue(value, m_highBlock));
8441     }
8442     void setInt52(Node* node, LValue value, Int52Kind kind)
8443     {
8444         switch (kind) {
8445         case Int52:
8446             setInt52(node, value);
8447             return;
8448
8449         case StrictInt52:
8450             setStrictInt52(node, value);
8451             return;
8452         }
8453
8454         DFG_CRASH(m_graph, m_node, "Corrupt int52 kind");
8455     }
8456     void setJSValue(Node* node, LValue value)
8457     {
8458         m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock));
8459     }
8460     void setBoolean(Node* node, LValue value)
8461     {
8462         m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock));
8463     }
8464     void setStorage(Node* node, LValue value)
8465     {
8466         m_storageValues.set(node, LoweredNodeValue(value, m_highBlock));
8467     }
8468     void setDouble(Node* node, LValue value)
8469     {
8470         m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock));
8471     }
8472
8473     void setInt32(LValue value)
8474     {
8475         setInt32(m_node, value);
8476     }
8477     void setInt52(LValue value)
8478     {
8479         setInt52(m_node, value);
8480     }
8481     void setStrictInt52(LValue value)
8482     {
8483         setStrictInt52(m_node, value);
8484     }
8485     void setInt52(LValue value, Int52Kind kind)
8486     {
8487         setInt52(m_node, value, kind);
8488     }
8489     void setJSValue(LValue value)
8490     {
8491         setJSValue(m_node, value);
8492     }
8493     void setBoolean(LValue value)
8494     {
8495         setBoolean(m_node, value);
8496     }
8497     void setStorage(LValue value)
8498     {
8499         setStorage(m_node, value);
8500     }
8501     void setDouble(LValue value)
8502     {
8503         setDouble(m_node, value);
8504     }
8505
8506     bool isValid(const LoweredNodeValue& value)
8507     {
8508         if (!value)
8509             return false;
8510         if (!m_graph.m_dominators.dominates(value.block(), m_highBlock))
8511             return false;
8512         return true;
8513     }
8514
8515     void addWeakReference(JSCell* target)
8516     {
8517         m_graph.m_plan.weakReferences.addLazily(target);
8518     }
8519
8520     LValue loadStructure(LValue value)
8521     {
8522         LValue tableIndex = m_out.load32(value, m_heaps.JSCell_structureID);
8523         LValue tableBase = m_out.loadPtr(
8524             m_out.absolute(vm().heap.structureIDTable().base()));
8525         TypedPointer address = m_out.baseIndex(
8526             m_heaps.structureTable, tableBase, m_out.zeroExtPtr(tableIndex));
8527         return m_out.loadPtr(address);
8528     }
8529
8530     LValue weakPointer(JSCell* pointer)
8531     {
8532         addWeakReference(pointer);
8533         return m_out.constIntPtr(pointer);
8534     }
8535
8536     LValue weakStructureID(Structure* structure)
8537     {
8538         addWeakReference(structure);
8539         return m_out.constInt32(structure->id());
8540     }
8541
8542     LValue weakStructure(Structure* structure)
8543     {
8544         return weakPointer(structure);
8545     }
8546
8547     TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0)
8548     {
8549         return m_out.address(base, m_heaps.variables[operand], offset);
8550     }
8551     TypedPointer payloadFor(LValue base, int operand)
8552     {
8553         return addressFor(base, operand, PayloadOffset);
8554     }
8555     TypedPointer tagFor(LValue base, int operand)
8556     {
8557         return addressFor(base, operand, TagOffset);
8558     }
8559     TypedPointer addressFor(int operand, ptrdiff_t offset = 0)
8560     {
8561         return addressFor(VirtualRegister(operand), offset);
8562     }
8563     TypedPointer addressFor(VirtualRegister operand, ptrdiff_t offset = 0)
8564     {
8565         if (operand.isLocal())
8566             return addressFor(m_captured, operand.offset(), offset);
8567         return addressFor(m_callFrame, operand.offset(), offset);
8568     }
8569     TypedPointer payloadFor(int operand)
8570     {
8571         return payloadFor(VirtualRegister(operand));
8572     }
8573     TypedPointer payloadFor(VirtualRegister operand)
8574     {
8575         return addressFor(operand, PayloadOffset);
8576     }
8577     TypedPointer tagFor(int operand)
8578     {
8579         return tagFor(VirtualRegister(operand));
8580     }
8581     TypedPointer tagFor(VirtualRegister operand)
8582     {
8583         return addressFor(operand, TagOffset);
8584     }
8585
8586     AbstractValue abstractValue(Node* node)
8587     {
8588         return m_state.forNode(node);
8589     }
8590     AbstractValue abstractValue(Edge edge)
8591     {
8592         return abstractValue(edge.node());
8593     }
8594
8595     SpeculatedType provenType(Node* node)
8596     {
8597         return abstractValue(node).m_type;
8598     }
8599     SpeculatedType provenType(Edge edge)
8600     {
8601         return provenType(edge.node());
8602     }
8603
8604     JSValue provenValue(Node* node)
8605     {
8606         return abstractValue(node).m_value;
8607     }
8608     JSValue provenValue(Edge edge)
8609     {
8610         return provenValue(edge.node());
8611     }
8612
8613     StructureAbstractValue abstractStructure(Node* node)
8614     {
8615         return abstractValue(node).m_structure;
8616     }
8617     StructureAbstractValue abstractStructure(Edge edge)
8618     {
8619         return abstractStructure(edge.node());
8620     }
8621
8622     void crash()
8623     {
8624         crash(m_highBlock->index, m_node->index());
8625     }
8626     void crash(BlockIndex blockIndex, unsigned nodeIndex)
8627     {
8628 #if ASSERT_DISABLED
8629         m_out.call(m_out.operation(ftlUnreachable));
8630         UNUSED_PARAM(blockIndex);
8631         UNUSED_PARAM(nodeIndex);
8632 #else
8633         m_out.call(
8634             m_out.intToPtr(
8635                 m_out.constIntPtr(ftlUnreachable),
8636                 pointerType(
8637                     functionType(
8638                         m_out.voidType, m_out.intPtr, m_out.int32, m_out.int32))),
8639             m_out.constIntPtr(codeBlock()), m_out.constInt32(blockIndex),
8640             m_out.constInt32(nodeIndex));
8641 #endif
8642         m_out.unreachable();
8643     }
8644
8645     AvailabilityMap& availabilityMap() { return m_availabilityCalculator.m_availability; }
8646
8647     VM& vm() { return m_graph.m_vm; }
8648     CodeBlock* codeBlock() { return m_graph.m_codeBlock; }
8649
8650     Graph& m_graph;
8651     State& m_ftlState;
8652     AbstractHeapRepository m_heaps;
8653     Output m_out;
8654
8655     LBasicBlock m_prologue;
8656     LBasicBlock m_handleExceptions;
8657     HashMap<BasicBlock*, LBasicBlock> m_blocks;
8658
8659     LValue m_execState;
8660     LValue m_execStorage;
8661     LValue m_callFrame;
8662     LValue m_captured;
8663     LValue m_tagTypeNumber;
8664     LValue m_tagMask;
8665
8666     HashMap<Node*, LoweredNodeValue> m_int32Values;
8667     HashMap<Node*, LoweredNodeValue> m_strictInt52Values;
8668     HashMap<Node*, LoweredNodeValue> m_int52Values;
8669     HashMap<Node*, LoweredNodeValue> m_jsValueValues;
8670     HashMap<Node*, LoweredNodeValue> m_booleanValues;
8671     HashMap<Node*, LoweredNodeValue> m_storageValues;
8672     HashMap<Node*, LoweredNodeValue> m_doubleValues;
8673
8674     // This is a bit of a hack. It prevents LLVM from having to do CSE on loading of arguments.
8675     // It's nice to have these optimizations on our end because we can guarantee them a bit better.
8676     // Probably also saves LLVM compile time.
8677     HashMap<Node*, LValue> m_loadedArgumentValues;
8678
8679     HashMap<Node*, LValue> m_phis;
8680
8681     LocalOSRAvailabilityCalculator m_availabilityCalculator;
8682
8683     Vector<AvailableRecovery, 3> m_availableRecoveries;
8684
8685     InPlaceAbstractState m_state;
8686     AbstractInterpreter<InPlaceAbstractState> m_interpreter;
8687     BasicBlock* m_highBlock;
8688     BasicBlock* m_nextHighBlock;
8689     LBasicBlock m_nextLowBlock;
8690
8691     CodeOrigin m_codeOriginForExitTarget;
8692     CodeOrigin m_codeOriginForExitProfile;
8693     unsigned m_nodeIndex;
8694     Node* m_node;
8695
8696     uint32_t m_stackmapIDs;
8697     unsigned m_tbaaKind;
8698     unsigned m_tbaaStructKind;
8699 };
8700
8701 } // anonymous namespace
8702
8703 void lowerDFGToLLVM(State& state)
8704 {
8705     LowerDFGToLLVM lowering(state);
8706     lowering.lower();
8707 }
8708
8709 } } // namespace JSC::FTL
8710
8711 #endif // ENABLE(FTL_JIT)
8712