dfg/DFGSpeculativeJIT32_64.cpp

   1 /*
   2  * Copyright (C) 2011, 2012, 2013 Apple Inc. All rights reserved.
   3  * Copyright (C) 2011 Intel Corporation. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  15  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  21  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  22  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  24  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include "config.h"
  28 #include "DFGSpeculativeJIT.h"
  29
  30 #if ENABLE(DFG_JIT)
  31
  32 #include "ArrayPrototype.h"
  33 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
  34 #include "DFGSlowPathGenerator.h"
  35 #include "JSActivation.h"
  36 #include "ObjectPrototype.h"
  37 #include "Operations.h"
  38
  39 namespace JSC { namespace DFG {
  40
  41 #if USE(JSVALUE32_64)
  42
  43 GPRReg SpeculativeJIT::fillInteger(Edge edge, DataFormat& returnFormat)
  44 {
  45     ASSERT(!needsTypeCheck(edge, SpecInt32));
  46
  47     VirtualRegister virtualRegister = edge->virtualRegister();
  48     GenerationInfo& info = m_generationInfo[virtualRegister];
  49
  50     if (info.registerFormat() == DataFormatNone) {
  51         GPRReg gpr = allocate();
  52
  53         if (edge->hasConstant()) {
  54             m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
  55             if (isInt32Constant(edge.node()))
  56                 m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
  57             else if (isNumberConstant(edge.node()))
  58                 RELEASE_ASSERT_NOT_REACHED();
  59             else {
  60                 ASSERT(isJSConstant(edge.node()));
  61                 JSValue jsValue = valueOfJSConstant(edge.node());
  62                 m_jit.move(MacroAssembler::Imm32(jsValue.payload()), gpr);
  63             }
  64         } else {
  65             ASSERT(info.spillFormat() == DataFormatJS || info.spillFormat() == DataFormatJSInteger || info.spillFormat() == DataFormatInteger);
  66             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
  67             m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr);
  68         }
  69
  70         info.fillInteger(*m_stream, gpr);
  71         returnFormat = DataFormatInteger;
  72         return gpr;
  73     }
  74
  75     switch (info.registerFormat()) {
  76     case DataFormatNone:
  77         // Should have filled, above.
  78     case DataFormatJSDouble:
  79     case DataFormatDouble:
  80     case DataFormatJS:
  81     case DataFormatCell:
  82     case DataFormatJSCell:
  83     case DataFormatBoolean:
  84     case DataFormatJSBoolean:
  85     case DataFormatStorage:
  86         // Should only be calling this function if we know this operand to be integer.
  87         RELEASE_ASSERT_NOT_REACHED();
  88
  89     case DataFormatJSInteger: {
  90         GPRReg tagGPR = info.tagGPR();
  91         GPRReg payloadGPR = info.payloadGPR();
  92         m_gprs.lock(tagGPR);
  93         m_jit.jitAssertIsJSInt32(tagGPR);
  94         m_gprs.unlock(tagGPR);
  95         m_gprs.lock(payloadGPR);
  96         m_gprs.release(tagGPR);
  97         m_gprs.release(payloadGPR);
  98         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderInteger);
  99         info.fillInteger(*m_stream, payloadGPR);
 100         returnFormat = DataFormatInteger;
 101         return payloadGPR;
 102     }
 103
 104     case DataFormatInteger: {
 105         GPRReg gpr = info.gpr();
 106         m_gprs.lock(gpr);
 107         m_jit.jitAssertIsInt32(gpr);
 108         returnFormat = DataFormatInteger;
 109         return gpr;
 110     }
 111
 112     default:
 113         RELEASE_ASSERT_NOT_REACHED();
 114         return InvalidGPRReg;
 115     }
 116 }
 117
 118 bool SpeculativeJIT::fillJSValue(Edge edge, GPRReg& tagGPR, GPRReg& payloadGPR, FPRReg& fpr)
 119 {
 120     // FIXME: For double we could fill with a FPR.
 121     UNUSED_PARAM(fpr);
 122
 123     VirtualRegister virtualRegister = edge->virtualRegister();
 124     GenerationInfo& info = m_generationInfo[virtualRegister];
 125
 126     switch (info.registerFormat()) {
 127     case DataFormatNone: {
 128
 129         if (edge->hasConstant()) {
 130             tagGPR = allocate();
 131             payloadGPR = allocate();
 132             m_jit.move(Imm32(valueOfJSConstant(edge.node()).tag()), tagGPR);
 133             m_jit.move(Imm32(valueOfJSConstant(edge.node()).payload()), payloadGPR);
 134             m_gprs.retain(tagGPR, virtualRegister, SpillOrderConstant);
 135             m_gprs.retain(payloadGPR, virtualRegister, SpillOrderConstant);
 136             info.fillJSValue(*m_stream, tagGPR, payloadGPR, isInt32Constant(edge.node()) ? DataFormatJSInteger : DataFormatJS);
 137         } else {
 138             DataFormat spillFormat = info.spillFormat();
 139             ASSERT(spillFormat != DataFormatNone && spillFormat != DataFormatStorage);
 140             tagGPR = allocate();
 141             payloadGPR = allocate();
 142             switch (spillFormat) {
 143             case DataFormatInteger:
 144                 m_jit.move(TrustedImm32(JSValue::Int32Tag), tagGPR);
 145                 spillFormat = DataFormatJSInteger; // This will be used as the new register format.
 146                 break;
 147             case DataFormatCell:
 148                 m_jit.move(TrustedImm32(JSValue::CellTag), tagGPR);
 149                 spillFormat = DataFormatJSCell; // This will be used as the new register format.
 150                 break;
 151             case DataFormatBoolean:
 152                 m_jit.move(TrustedImm32(JSValue::BooleanTag), tagGPR);
 153                 spillFormat = DataFormatJSBoolean; // This will be used as the new register format.
 154                 break;
 155             default:
 156                 m_jit.load32(JITCompiler::tagFor(virtualRegister), tagGPR);
 157                 break;
 158             }
 159             m_jit.load32(JITCompiler::payloadFor(virtualRegister), payloadGPR);
 160             m_gprs.retain(tagGPR, virtualRegister, SpillOrderSpilled);
 161             m_gprs.retain(payloadGPR, virtualRegister, SpillOrderSpilled);
 162             info.fillJSValue(*m_stream, tagGPR, payloadGPR, spillFormat == DataFormatJSDouble ? DataFormatJS : spillFormat);
 163         }
 164
 165         return true;
 166     }
 167
 168     case DataFormatInteger:
 169     case DataFormatCell:
 170     case DataFormatBoolean: {
 171         GPRReg gpr = info.gpr();
 172         // If the register has already been locked we need to take a copy.
 173         if (m_gprs.isLocked(gpr)) {
 174             payloadGPR = allocate();
 175             m_jit.move(gpr, payloadGPR);
 176         } else {
 177             payloadGPR = gpr;
 178             m_gprs.lock(gpr);
 179         }
 180         tagGPR = allocate();
 181         uint32_t tag = JSValue::EmptyValueTag;
 182         DataFormat fillFormat = DataFormatJS;
 183         switch (info.registerFormat()) {
 184         case DataFormatInteger:
 185             tag = JSValue::Int32Tag;
 186             fillFormat = DataFormatJSInteger;
 187             break;
 188         case DataFormatCell:
 189             tag = JSValue::CellTag;
 190             fillFormat = DataFormatJSCell;
 191             break;
 192         case DataFormatBoolean:
 193             tag = JSValue::BooleanTag;
 194             fillFormat = DataFormatJSBoolean;
 195             break;
 196         default:
 197             RELEASE_ASSERT_NOT_REACHED();
 198             break;
 199         }
 200         m_jit.move(TrustedImm32(tag), tagGPR);
 201         m_gprs.release(gpr);
 202         m_gprs.retain(tagGPR, virtualRegister, SpillOrderJS);
 203         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderJS);
 204         info.fillJSValue(*m_stream, tagGPR, payloadGPR, fillFormat);
 205         return true;
 206     }
 207
 208     case DataFormatJSDouble:
 209     case DataFormatDouble: {
 210         FPRReg oldFPR = info.fpr();
 211         m_fprs.lock(oldFPR);
 212         tagGPR = allocate();
 213         payloadGPR = allocate();
 214         boxDouble(oldFPR, tagGPR, payloadGPR);
 215         m_fprs.unlock(oldFPR);
 216         m_fprs.release(oldFPR);
 217         m_gprs.retain(tagGPR, virtualRegister, SpillOrderJS);
 218         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderJS);
 219         info.fillJSValue(*m_stream, tagGPR, payloadGPR, DataFormatJS);
 220         return true;
 221     }
 222
 223     case DataFormatJS:
 224     case DataFormatJSInteger:
 225     case DataFormatJSCell:
 226     case DataFormatJSBoolean: {
 227         tagGPR = info.tagGPR();
 228         payloadGPR = info.payloadGPR();
 229         m_gprs.lock(tagGPR);
 230         m_gprs.lock(payloadGPR);
 231         return true;
 232     }
 233
 234     case DataFormatStorage:
 235         // this type currently never occurs
 236         RELEASE_ASSERT_NOT_REACHED();
 237
 238     default:
 239         RELEASE_ASSERT_NOT_REACHED();
 240         return true;
 241     }
 242 }
 243
 244 void SpeculativeJIT::nonSpeculativeUInt32ToNumber(Node* node)
 245 {
 246     IntegerOperand op1(this, node->child1());
 247     FPRTemporary boxer(this);
 248     GPRTemporary resultTag(this, op1);
 249     GPRTemporary resultPayload(this);
 250
 251     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, op1.gpr(), TrustedImm32(0));
 252
 253     m_jit.convertInt32ToDouble(op1.gpr(), boxer.fpr());
 254     m_jit.move(JITCompiler::TrustedImmPtr(&AssemblyHelpers::twoToThe32), resultPayload.gpr()); // reuse resultPayload register here.
 255     m_jit.addDouble(JITCompiler::Address(resultPayload.gpr(), 0), boxer.fpr());
 256
 257     boxDouble(boxer.fpr(), resultTag.gpr(), resultPayload.gpr());
 258
 259     JITCompiler::Jump done = m_jit.jump();
 260
 261     positive.link(&m_jit);
 262
 263     m_jit.move(TrustedImm32(JSValue::Int32Tag), resultTag.gpr());
 264     m_jit.move(op1.gpr(), resultPayload.gpr());
 265
 266     done.link(&m_jit);
 267
 268     jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
 269 }
 270
 271 void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseTagGPROrNone, GPRReg basePayloadGPR, GPRReg resultTagGPR, GPRReg resultPayloadGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode)
 272 {
 273     JITCompiler::DataLabelPtr structureToCompare;
 274     JITCompiler::PatchableJump structureCheck = m_jit.patchableBranchPtrWithPatch(JITCompiler::NotEqual, JITCompiler::Address(basePayloadGPR, JSCell::structureOffset()), structureToCompare, JITCompiler::TrustedImmPtr(reinterpret_cast<void*>(unusedPointer)));
 275
 276     JITCompiler::ConvertibleLoadLabel propertyStorageLoad = m_jit.convertibleLoadPtr(JITCompiler::Address(basePayloadGPR, JSObject::butterflyOffset()), resultPayloadGPR);
 277     JITCompiler::DataLabelCompact tagLoadWithPatch = m_jit.load32WithCompactAddressOffsetPatch(JITCompiler::Address(resultPayloadGPR, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTagGPR);
 278     JITCompiler::DataLabelCompact payloadLoadWithPatch = m_jit.load32WithCompactAddressOffsetPatch(JITCompiler::Address(resultPayloadGPR, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR);
 279
 280     JITCompiler::Label doneLabel = m_jit.label();
 281
 282     OwnPtr<SlowPathGenerator> slowPath;
 283     if (baseTagGPROrNone == InvalidGPRReg) {
 284         if (!slowPathTarget.isSet()) {
 285             slowPath = slowPathCall(
 286                 structureCheck.m_jump, this, operationGetByIdOptimize,
 287                 JSValueRegs(resultTagGPR, resultPayloadGPR),
 288                 static_cast<int32_t>(JSValue::CellTag), basePayloadGPR,
 289                 identifier(identifierNumber));
 290         } else {
 291             JITCompiler::JumpList slowCases;
 292             slowCases.append(structureCheck.m_jump);
 293             slowCases.append(slowPathTarget);
 294             slowPath = slowPathCall(
 295                 slowCases, this, operationGetByIdOptimize,
 296                 JSValueRegs(resultTagGPR, resultPayloadGPR),
 297                 static_cast<int32_t>(JSValue::CellTag), basePayloadGPR,
 298                 identifier(identifierNumber));
 299         }
 300     } else {
 301         if (!slowPathTarget.isSet()) {
 302             slowPath = slowPathCall(
 303                 structureCheck.m_jump, this, operationGetByIdOptimize,
 304                 JSValueRegs(resultTagGPR, resultPayloadGPR), baseTagGPROrNone, basePayloadGPR,
 305                 identifier(identifierNumber));
 306         } else {
 307             JITCompiler::JumpList slowCases;
 308             slowCases.append(structureCheck.m_jump);
 309             slowCases.append(slowPathTarget);
 310             slowPath = slowPathCall(
 311                 slowCases, this, operationGetByIdOptimize,
 312                 JSValueRegs(resultTagGPR, resultPayloadGPR), baseTagGPROrNone, basePayloadGPR,
 313                 identifier(identifierNumber));
 314         }
 315     }
 316     m_jit.addPropertyAccess(
 317         PropertyAccessRecord(
 318             codeOrigin, structureToCompare, structureCheck, propertyStorageLoad,
 319             tagLoadWithPatch, payloadLoadWithPatch, slowPath.get(), doneLabel,
 320             safeCast<int8_t>(basePayloadGPR), safeCast<int8_t>(resultTagGPR),
 321             safeCast<int8_t>(resultPayloadGPR), usedRegisters(),
 322             spillMode == NeedToSpill ? PropertyAccessRecord::RegistersInUse : PropertyAccessRecord::RegistersFlushed));
 323     addSlowPathGenerator(slowPath.release());
 324 }
 325
 326 void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg basePayloadGPR, GPRReg valueTagGPR, GPRReg valuePayloadGPR, Edge valueUse, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget)
 327 {
 328     JITCompiler::DataLabelPtr structureToCompare;
 329     JITCompiler::PatchableJump structureCheck = m_jit.patchableBranchPtrWithPatch(JITCompiler::NotEqual, JITCompiler::Address(basePayloadGPR, JSCell::structureOffset()), structureToCompare, JITCompiler::TrustedImmPtr(reinterpret_cast<void*>(unusedPointer)));
 330
 331     writeBarrier(basePayloadGPR, valueTagGPR, valueUse, WriteBarrierForPropertyAccess, scratchGPR);
 332
 333     JITCompiler::ConvertibleLoadLabel propertyStorageLoad = m_jit.convertibleLoadPtr(JITCompiler::Address(basePayloadGPR, JSObject::butterflyOffset()), scratchGPR);
 334     JITCompiler::DataLabel32 tagStoreWithPatch = m_jit.store32WithAddressOffsetPatch(valueTagGPR, JITCompiler::Address(scratchGPR, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
 335     JITCompiler::DataLabel32 payloadStoreWithPatch = m_jit.store32WithAddressOffsetPatch(valuePayloadGPR, JITCompiler::Address(scratchGPR, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
 336
 337     JITCompiler::Label doneLabel = m_jit.label();
 338     V_DFGOperation_EJCI optimizedCall;
 339     if (m_jit.strictModeFor(m_currentNode->codeOrigin)) {
 340         if (putKind == Direct)
 341             optimizedCall = operationPutByIdDirectStrictOptimize;
 342         else
 343             optimizedCall = operationPutByIdStrictOptimize;
 344     } else {
 345         if (putKind == Direct)
 346             optimizedCall = operationPutByIdDirectNonStrictOptimize;
 347         else
 348             optimizedCall = operationPutByIdNonStrictOptimize;
 349     }
 350     OwnPtr<SlowPathGenerator> slowPath;
 351     if (!slowPathTarget.isSet()) {
 352         slowPath = slowPathCall(
 353             structureCheck.m_jump, this, optimizedCall, NoResult, valueTagGPR, valuePayloadGPR,
 354             basePayloadGPR, identifier(identifierNumber));
 355     } else {
 356         JITCompiler::JumpList slowCases;
 357         slowCases.append(structureCheck.m_jump);
 358         slowCases.append(slowPathTarget);
 359         slowPath = slowPathCall(
 360             slowCases, this, optimizedCall, NoResult, valueTagGPR, valuePayloadGPR,
 361             basePayloadGPR, identifier(identifierNumber));
 362     }
 363     RegisterSet currentlyUsedRegisters = usedRegisters();
 364     currentlyUsedRegisters.clear(scratchGPR);
 365     ASSERT(currentlyUsedRegisters.get(basePayloadGPR));
 366     ASSERT(currentlyUsedRegisters.get(valueTagGPR));
 367     ASSERT(currentlyUsedRegisters.get(valuePayloadGPR));
 368     m_jit.addPropertyAccess(
 369         PropertyAccessRecord(
 370             codeOrigin, structureToCompare, structureCheck, propertyStorageLoad,
 371             JITCompiler::DataLabelCompact(tagStoreWithPatch.label()),
 372             JITCompiler::DataLabelCompact(payloadStoreWithPatch.label()),
 373             slowPath.get(), doneLabel, safeCast<int8_t>(basePayloadGPR),
 374             safeCast<int8_t>(valueTagGPR), safeCast<int8_t>(valuePayloadGPR),
 375             usedRegisters()));
 376     addSlowPathGenerator(slowPath.release());
 377 }
 378
 379 void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool invert)
 380 {
 381     JSValueOperand arg(this, operand);
 382     GPRReg argTagGPR = arg.tagGPR();
 383     GPRReg argPayloadGPR = arg.payloadGPR();
 384
 385     GPRTemporary resultPayload(this, arg, false);
 386     GPRReg resultPayloadGPR = resultPayload.gpr();
 387
 388     JITCompiler::Jump notCell;
 389     JITCompiler::Jump notMasqueradesAsUndefined;
 390     if (m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
 391         if (!isKnownCell(operand.node()))
 392             notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag));
 393
 394         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
 395         m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultPayloadGPR);
 396         notMasqueradesAsUndefined = m_jit.jump();
 397     } else {
 398         GPRTemporary localGlobalObject(this);
 399         GPRTemporary remoteGlobalObject(this);
 400
 401         if (!isKnownCell(operand.node()))
 402             notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag));
 403
 404         m_jit.loadPtr(JITCompiler::Address(argPayloadGPR, JSCell::structureOffset()), resultPayloadGPR);
 405         JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(resultPayloadGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined));
 406
 407         m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultPayloadGPR);
 408         notMasqueradesAsUndefined = m_jit.jump();
 409
 410         isMasqueradesAsUndefined.link(&m_jit);
 411         GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
 412         GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
 413         m_jit.move(JITCompiler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)), localGlobalObjectGPR);
 414         m_jit.loadPtr(JITCompiler::Address(resultPayloadGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR);
 415         m_jit.compare32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultPayloadGPR);
 416     }
 417
 418     if (!isKnownCell(operand.node())) {
 419         JITCompiler::Jump done = m_jit.jump();
 420
 421         notCell.link(&m_jit);
 422         // null or undefined?
 423         COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag);
 424         m_jit.move(argTagGPR, resultPayloadGPR);
 425         m_jit.or32(TrustedImm32(1), resultPayloadGPR);
 426         m_jit.compare32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultPayloadGPR, TrustedImm32(JSValue::NullTag), resultPayloadGPR);
 427
 428         done.link(&m_jit);
 429     }
 430
 431     notMasqueradesAsUndefined.link(&m_jit);
 432
 433     booleanResult(resultPayloadGPR, m_currentNode);
 434 }
 435
 436 void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branchNode, bool invert)
 437 {
 438     BlockIndex taken = branchNode->takenBlockIndex();
 439     BlockIndex notTaken = branchNode->notTakenBlockIndex();
 440
 441     if (taken == nextBlock()) {
 442         invert = !invert;
 443         BlockIndex tmp = taken;
 444         taken = notTaken;
 445         notTaken = tmp;
 446     }
 447
 448     JSValueOperand arg(this, operand);
 449     GPRReg argTagGPR = arg.tagGPR();
 450     GPRReg argPayloadGPR = arg.payloadGPR();
 451
 452     GPRTemporary result(this, arg);
 453     GPRReg resultGPR = result.gpr();
 454
 455     JITCompiler::Jump notCell;
 456
 457     if (m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
 458         if (!isKnownCell(operand.node()))
 459             notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag));
 460
 461         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
 462         jump(invert ? taken : notTaken, ForceJump);
 463     } else {
 464         GPRTemporary localGlobalObject(this);
 465         GPRTemporary remoteGlobalObject(this);
 466
 467         if (!isKnownCell(operand.node()))
 468             notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag));
 469
 470         m_jit.loadPtr(JITCompiler::Address(argPayloadGPR, JSCell::structureOffset()), resultGPR);
 471         branchTest8(JITCompiler::Zero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined), invert ? taken : notTaken);
 472
 473         GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
 474         GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
 475         m_jit.move(TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)), localGlobalObjectGPR);
 476         m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR);
 477         branchPtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, invert ? notTaken : taken);
 478     }
 479
 480     if (!isKnownCell(operand.node())) {
 481         jump(notTaken, ForceJump);
 482
 483         notCell.link(&m_jit);
 484         // null or undefined?
 485         COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag);
 486         m_jit.move(argTagGPR, resultGPR);
 487         m_jit.or32(TrustedImm32(1), resultGPR);
 488         branch32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm32(JSValue::NullTag), taken);
 489     }
 490
 491     jump(notTaken);
 492 }
 493
 494 bool SpeculativeJIT::nonSpeculativeCompareNull(Node* node, Edge operand, bool invert)
 495 {
 496     unsigned branchIndexInBlock = detectPeepHoleBranch();
 497     if (branchIndexInBlock != UINT_MAX) {
 498         Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock);
 499
 500         ASSERT(node->adjustedRefCount() == 1);
 501
 502         nonSpeculativePeepholeBranchNull(operand, branchNode, invert);
 503
 504         use(node->child1());
 505         use(node->child2());
 506         m_indexInBlock = branchIndexInBlock;
 507         m_currentNode = branchNode;
 508
 509         return true;
 510     }
 511
 512     nonSpeculativeNonPeepholeCompareNull(operand, invert);
 513
 514     return false;
 515 }
 516
 517 void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction)
 518 {
 519     BlockIndex taken = branchNode->takenBlockIndex();
 520     BlockIndex notTaken = branchNode->notTakenBlockIndex();
 521
 522     JITCompiler::ResultCondition callResultCondition = JITCompiler::NonZero;
 523
 524     // The branch instruction will branch to the taken block.
 525     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
 526     if (taken == nextBlock()) {
 527         cond = JITCompiler::invert(cond);
 528         callResultCondition = JITCompiler::Zero;
 529         BlockIndex tmp = taken;
 530         taken = notTaken;
 531         notTaken = tmp;
 532     }
 533
 534     JSValueOperand arg1(this, node->child1());
 535     JSValueOperand arg2(this, node->child2());
 536     GPRReg arg1TagGPR = arg1.tagGPR();
 537     GPRReg arg1PayloadGPR = arg1.payloadGPR();
 538     GPRReg arg2TagGPR = arg2.tagGPR();
 539     GPRReg arg2PayloadGPR = arg2.payloadGPR();
 540
 541     JITCompiler::JumpList slowPath;
 542
 543     if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
 544         GPRResult result(this);
 545         GPRReg resultGPR = result.gpr();
 546
 547         arg1.use();
 548         arg2.use();
 549
 550         flushRegisters();
 551         callOperation(helperFunction, resultGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 552
 553         branchTest32(callResultCondition, resultGPR, taken);
 554     } else {
 555         GPRTemporary result(this);
 556         GPRReg resultGPR = result.gpr();
 557
 558         arg1.use();
 559         arg2.use();
 560
 561         if (!isKnownInteger(node->child1().node()))
 562             slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, arg1TagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)));
 563         if (!isKnownInteger(node->child2().node()))
 564             slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, arg2TagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)));
 565
 566         branch32(cond, arg1PayloadGPR, arg2PayloadGPR, taken);
 567
 568         if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
 569             jump(notTaken, ForceJump);
 570
 571             slowPath.link(&m_jit);
 572
 573             silentSpillAllRegisters(resultGPR);
 574             callOperation(helperFunction, resultGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 575             silentFillAllRegisters(resultGPR);
 576
 577             branchTest32(callResultCondition, resultGPR, taken);
 578         }
 579     }
 580
 581     jump(notTaken);
 582
 583     m_indexInBlock = m_jit.graph().m_blocks[m_block]->size() - 1;
 584     m_currentNode = branchNode;
 585 }
 586
 587 template<typename JumpType>
 588 class CompareAndBoxBooleanSlowPathGenerator
 589     : public CallSlowPathGenerator<JumpType, S_DFGOperation_EJJ, GPRReg> {
 590 public:
 591     CompareAndBoxBooleanSlowPathGenerator(
 592         JumpType from, SpeculativeJIT* jit,
 593         S_DFGOperation_EJJ function, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload,
 594         GPRReg arg2Tag, GPRReg arg2Payload)
 595         : CallSlowPathGenerator<JumpType, S_DFGOperation_EJJ, GPRReg>(
 596             from, jit, function, NeedToSpill, result)
 597         , m_arg1Tag(arg1Tag)
 598         , m_arg1Payload(arg1Payload)
 599         , m_arg2Tag(arg2Tag)
 600         , m_arg2Payload(arg2Payload)
 601     {
 602     }
 603
 604 protected:
 605     virtual void generateInternal(SpeculativeJIT* jit)
 606     {
 607         this->setUp(jit);
 608         this->recordCall(
 609             jit->callOperation(
 610                 this->m_function, this->m_result, m_arg1Tag, m_arg1Payload, m_arg2Tag,
 611                 m_arg2Payload));
 612         jit->m_jit.and32(JITCompiler::TrustedImm32(1), this->m_result);
 613         this->tearDown(jit);
 614     }
 615
 616 private:
 617     GPRReg m_arg1Tag;
 618     GPRReg m_arg1Payload;
 619     GPRReg m_arg2Tag;
 620     GPRReg m_arg2Payload;
 621 };
 622
 623 void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction)
 624 {
 625     JSValueOperand arg1(this, node->child1());
 626     JSValueOperand arg2(this, node->child2());
 627     GPRReg arg1TagGPR = arg1.tagGPR();
 628     GPRReg arg1PayloadGPR = arg1.payloadGPR();
 629     GPRReg arg2TagGPR = arg2.tagGPR();
 630     GPRReg arg2PayloadGPR = arg2.payloadGPR();
 631
 632     JITCompiler::JumpList slowPath;
 633
 634     if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
 635         GPRResult result(this);
 636         GPRReg resultPayloadGPR = result.gpr();
 637
 638         arg1.use();
 639         arg2.use();
 640
 641         flushRegisters();
 642         callOperation(helperFunction, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 643
 644         booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly);
 645     } else {
 646         GPRTemporary resultPayload(this, arg1, false);
 647         GPRReg resultPayloadGPR = resultPayload.gpr();
 648
 649         arg1.use();
 650         arg2.use();
 651
 652         if (!isKnownInteger(node->child1().node()))
 653             slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, arg1TagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)));
 654         if (!isKnownInteger(node->child2().node()))
 655             slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, arg2TagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)));
 656
 657         m_jit.compare32(cond, arg1PayloadGPR, arg2PayloadGPR, resultPayloadGPR);
 658
 659         if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
 660             addSlowPathGenerator(adoptPtr(
 661                 new CompareAndBoxBooleanSlowPathGenerator<JITCompiler::JumpList>(
 662                     slowPath, this, helperFunction, resultPayloadGPR, arg1TagGPR,
 663                     arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR)));
 664         }
 665
 666         booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly);
 667     }
 668 }
 669
 670 void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode, bool invert)
 671 {
 672     BlockIndex taken = branchNode->takenBlockIndex();
 673     BlockIndex notTaken = branchNode->notTakenBlockIndex();
 674
 675     // The branch instruction will branch to the taken block.
 676     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
 677     if (taken == nextBlock()) {
 678         invert = !invert;
 679         BlockIndex tmp = taken;
 680         taken = notTaken;
 681         notTaken = tmp;
 682     }
 683
 684     JSValueOperand arg1(this, node->child1());
 685     JSValueOperand arg2(this, node->child2());
 686     GPRReg arg1TagGPR = arg1.tagGPR();
 687     GPRReg arg1PayloadGPR = arg1.payloadGPR();
 688     GPRReg arg2TagGPR = arg2.tagGPR();
 689     GPRReg arg2PayloadGPR = arg2.payloadGPR();
 690
 691     GPRTemporary resultPayload(this, arg1, false);
 692     GPRReg resultPayloadGPR = resultPayload.gpr();
 693
 694     arg1.use();
 695     arg2.use();
 696
 697     if (isKnownCell(node->child1().node()) && isKnownCell(node->child2().node())) {
 698         // see if we get lucky: if the arguments are cells and they reference the same
 699         // cell, then they must be strictly equal.
 700         branchPtr(JITCompiler::Equal, arg1PayloadGPR, arg2PayloadGPR, invert ? notTaken : taken);
 701
 702         silentSpillAllRegisters(resultPayloadGPR);
 703         callOperation(operationCompareStrictEqCell, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 704         silentFillAllRegisters(resultPayloadGPR);
 705
 706         branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultPayloadGPR, taken);
 707     } else {
 708         // FIXME: Add fast paths for twoCells, number etc.
 709
 710         silentSpillAllRegisters(resultPayloadGPR);
 711         callOperation(operationCompareStrictEq, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 712         silentFillAllRegisters(resultPayloadGPR);
 713
 714         branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultPayloadGPR, taken);
 715     }
 716
 717     jump(notTaken);
 718 }
 719
 720 void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert)
 721 {
 722     JSValueOperand arg1(this, node->child1());
 723     JSValueOperand arg2(this, node->child2());
 724     GPRReg arg1TagGPR = arg1.tagGPR();
 725     GPRReg arg1PayloadGPR = arg1.payloadGPR();
 726     GPRReg arg2TagGPR = arg2.tagGPR();
 727     GPRReg arg2PayloadGPR = arg2.payloadGPR();
 728
 729     GPRTemporary resultPayload(this, arg1, false);
 730     GPRReg resultPayloadGPR = resultPayload.gpr();
 731
 732     arg1.use();
 733     arg2.use();
 734
 735     if (isKnownCell(node->child1().node()) && isKnownCell(node->child2().node())) {
 736         // see if we get lucky: if the arguments are cells and they reference the same
 737         // cell, then they must be strictly equal.
 738         // FIXME: this should flush registers instead of silent spill/fill.
 739         JITCompiler::Jump notEqualCase = m_jit.branchPtr(JITCompiler::NotEqual, arg1PayloadGPR, arg2PayloadGPR);
 740
 741         m_jit.move(JITCompiler::TrustedImm32(!invert), resultPayloadGPR);
 742         JITCompiler::Jump done = m_jit.jump();
 743
 744         notEqualCase.link(&m_jit);
 745
 746         silentSpillAllRegisters(resultPayloadGPR);
 747         callOperation(operationCompareStrictEqCell, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 748         silentFillAllRegisters(resultPayloadGPR);
 749
 750         m_jit.andPtr(JITCompiler::TrustedImm32(1), resultPayloadGPR);
 751
 752         done.link(&m_jit);
 753     } else {
 754         // FIXME: Add fast paths.
 755
 756         silentSpillAllRegisters(resultPayloadGPR);
 757         callOperation(operationCompareStrictEq, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 758         silentFillAllRegisters(resultPayloadGPR);
 759
 760         m_jit.andPtr(JITCompiler::TrustedImm32(1), resultPayloadGPR);
 761     }
 762
 763     booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly);
 764 }
 765
 766 void SpeculativeJIT::emitCall(Node* node)
 767 {
 768     if (node->op() != Call)
 769         ASSERT(node->op() == Construct);
 770
 771     // For constructors, the this argument is not passed but we have to make space
 772     // for it.
 773     int dummyThisArgument = node->op() == Call ? 0 : 1;
 774
 775     CallLinkInfo::CallType callType = node->op() == Call ? CallLinkInfo::Call : CallLinkInfo::Construct;
 776
 777     Edge calleeEdge = m_jit.graph().m_varArgChildren[node->firstChild()];
 778     JSValueOperand callee(this, calleeEdge);
 779     GPRReg calleeTagGPR = callee.tagGPR();
 780     GPRReg calleePayloadGPR = callee.payloadGPR();
 781     use(calleeEdge);
 782
 783     // The call instruction's first child is either the function (normal call) or the
 784     // receiver (method call). subsequent children are the arguments.
 785     int numPassedArgs = node->numChildren() - 1;
 786
 787     m_jit.store32(MacroAssembler::TrustedImm32(numPassedArgs + dummyThisArgument), callFramePayloadSlot(JSStack::ArgumentCount));
 788     m_jit.storePtr(GPRInfo::callFrameRegister, callFramePayloadSlot(JSStack::CallerFrame));
 789     m_jit.store32(calleePayloadGPR, callFramePayloadSlot(JSStack::Callee));
 790     m_jit.store32(calleeTagGPR, callFrameTagSlot(JSStack::Callee));
 791
 792     for (int i = 0; i < numPassedArgs; i++) {
 793         Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i];
 794         JSValueOperand arg(this, argEdge);
 795         GPRReg argTagGPR = arg.tagGPR();
 796         GPRReg argPayloadGPR = arg.payloadGPR();
 797         use(argEdge);
 798
 799         m_jit.store32(argTagGPR, argumentTagSlot(i + dummyThisArgument));
 800         m_jit.store32(argPayloadGPR, argumentPayloadSlot(i + dummyThisArgument));
 801     }
 802
 803     flushRegisters();
 804
 805     GPRResult resultPayload(this);
 806     GPRResult2 resultTag(this);
 807     GPRReg resultPayloadGPR = resultPayload.gpr();
 808     GPRReg resultTagGPR = resultTag.gpr();
 809
 810     JITCompiler::DataLabelPtr targetToCheck;
 811     JITCompiler::JumpList slowPath;
 812
 813     CallBeginToken token;
 814     m_jit.beginCall(node->codeOrigin, token);
 815
 816     m_jit.addPtr(TrustedImm32(m_jit.codeBlock()->m_numCalleeRegisters * sizeof(Register)), GPRInfo::callFrameRegister);
 817
 818     slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, calleeTagGPR, TrustedImm32(JSValue::CellTag)));
 819     slowPath.append(m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleePayloadGPR, targetToCheck));
 820     m_jit.loadPtr(MacroAssembler::Address(calleePayloadGPR, OBJECT_OFFSETOF(JSFunction, m_scope)), resultPayloadGPR);
 821     m_jit.storePtr(resultPayloadGPR, MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
 822     m_jit.store32(MacroAssembler::TrustedImm32(JSValue::CellTag), MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
 823
 824     CodeOrigin codeOrigin = node->codeOrigin;
 825     JITCompiler::Call fastCall = m_jit.nearCall();
 826     m_jit.notifyCall(fastCall, codeOrigin, token);
 827
 828     JITCompiler::Jump done = m_jit.jump();
 829
 830     slowPath.link(&m_jit);
 831
 832     if (calleeTagGPR == GPRInfo::nonArgGPR0) {
 833         if (calleePayloadGPR == GPRInfo::nonArgGPR1)
 834             m_jit.swap(GPRInfo::nonArgGPR1, GPRInfo::nonArgGPR0);
 835         else {
 836             m_jit.move(calleeTagGPR, GPRInfo::nonArgGPR1);
 837             m_jit.move(calleePayloadGPR, GPRInfo::nonArgGPR0);
 838         }
 839     } else {
 840         m_jit.move(calleePayloadGPR, GPRInfo::nonArgGPR0);
 841         m_jit.move(calleeTagGPR, GPRInfo::nonArgGPR1);
 842     }
 843     m_jit.prepareForExceptionCheck();
 844     JITCompiler::Call slowCall = m_jit.nearCall();
 845     m_jit.notifyCall(slowCall, codeOrigin, token);
 846
 847     done.link(&m_jit);
 848
 849     m_jit.setupResults(resultPayloadGPR, resultTagGPR);
 850
 851     jsValueResult(resultTagGPR, resultPayloadGPR, node, DataFormatJS, UseChildrenCalledExplicitly);
 852
 853     m_jit.addJSCall(fastCall, slowCall, targetToCheck, callType, calleePayloadGPR, node->codeOrigin);
 854 }
 855
 856 template<bool strict>
 857 GPRReg SpeculativeJIT::fillSpeculateIntInternal(Edge edge, DataFormat& returnFormat)
 858 {
 859 #if DFG_ENABLE(DEBUG_VERBOSE)
 860     dataLogF("SpecInt@%d   ", edge->index());
 861 #endif
 862     AbstractValue& value = m_state.forNode(edge);
 863     SpeculatedType type = value.m_type;
 864     ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32));
 865     value.filter(SpecInt32);
 866     VirtualRegister virtualRegister = edge->virtualRegister();
 867     GenerationInfo& info = m_generationInfo[virtualRegister];
 868
 869     switch (info.registerFormat()) {
 870     case DataFormatNone: {
 871         if ((edge->hasConstant() && !isInt32Constant(edge.node())) || info.spillFormat() == DataFormatDouble) {
 872             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 873             returnFormat = DataFormatInteger;
 874             return allocate();
 875         }
 876
 877         if (edge->hasConstant()) {
 878             ASSERT(isInt32Constant(edge.node()));
 879             GPRReg gpr = allocate();
 880             m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
 881             m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
 882             info.fillInteger(*m_stream, gpr);
 883             returnFormat = DataFormatInteger;
 884             return gpr;
 885         }
 886
 887         DataFormat spillFormat = info.spillFormat();
 888         ASSERT_UNUSED(spillFormat, (spillFormat & DataFormatJS) || spillFormat == DataFormatInteger);
 889
 890         // If we know this was spilled as an integer we can fill without checking.
 891         if (type & ~SpecInt32)
 892             speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
 893
 894         GPRReg gpr = allocate();
 895         m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr);
 896         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
 897         info.fillInteger(*m_stream, gpr);
 898         returnFormat = DataFormatInteger;
 899         return gpr;
 900     }
 901
 902     case DataFormatJSInteger:
 903     case DataFormatJS: {
 904         // Check the value is an integer.
 905         GPRReg tagGPR = info.tagGPR();
 906         GPRReg payloadGPR = info.payloadGPR();
 907         m_gprs.lock(tagGPR);
 908         m_gprs.lock(payloadGPR);
 909         if (type & ~SpecInt32)
 910             speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::Int32Tag)));
 911         m_gprs.unlock(tagGPR);
 912         m_gprs.release(tagGPR);
 913         m_gprs.release(payloadGPR);
 914         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderInteger);
 915         info.fillInteger(*m_stream, payloadGPR);
 916         // If !strict we're done, return.
 917         returnFormat = DataFormatInteger;
 918         return payloadGPR;
 919     }
 920
 921     case DataFormatInteger: {
 922         GPRReg gpr = info.gpr();
 923         m_gprs.lock(gpr);
 924         returnFormat = DataFormatInteger;
 925         return gpr;
 926     }
 927
 928     case DataFormatDouble:
 929     case DataFormatCell:
 930     case DataFormatBoolean:
 931     case DataFormatJSDouble:
 932     case DataFormatJSCell:
 933     case DataFormatJSBoolean:
 934         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 935         returnFormat = DataFormatInteger;
 936         return allocate();
 937
 938     case DataFormatStorage:
 939     default:
 940         RELEASE_ASSERT_NOT_REACHED();
 941         return InvalidGPRReg;
 942     }
 943 }
 944
 945 GPRReg SpeculativeJIT::fillSpeculateInt(Edge edge, DataFormat& returnFormat)
 946 {
 947     return fillSpeculateIntInternal<false>(edge, returnFormat);
 948 }
 949
 950 GPRReg SpeculativeJIT::fillSpeculateIntStrict(Edge edge)
 951 {
 952     DataFormat mustBeDataFormatInteger;
 953     GPRReg result = fillSpeculateIntInternal<true>(edge, mustBeDataFormatInteger);
 954     ASSERT(mustBeDataFormatInteger == DataFormatInteger);
 955     return result;
 956 }
 957
 958 FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge)
 959 {
 960 #if DFG_ENABLE(DEBUG_VERBOSE)
 961     dataLogF("SpecDouble@%d   ", edge->index());
 962 #endif
 963     AbstractValue& value = m_state.forNode(edge);
 964     SpeculatedType type = value.m_type;
 965     ASSERT(edge.useKind() != KnownNumberUse || !(value.m_type & ~SpecNumber));
 966     value.filter(SpecNumber);
 967     VirtualRegister virtualRegister = edge->virtualRegister();
 968     GenerationInfo& info = m_generationInfo[virtualRegister];
 969
 970     if (info.registerFormat() == DataFormatNone) {
 971
 972         if (edge->hasConstant()) {
 973             if (isInt32Constant(edge.node())) {
 974                 GPRReg gpr = allocate();
 975                 m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
 976                 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
 977                 info.fillInteger(*m_stream, gpr);
 978                 unlock(gpr);
 979             } else if (isNumberConstant(edge.node())) {
 980                 FPRReg fpr = fprAllocate();
 981                 m_jit.loadDouble(addressOfDoubleConstant(edge.node()), fpr);
 982                 m_fprs.retain(fpr, virtualRegister, SpillOrderConstant);
 983                 info.fillDouble(*m_stream, fpr);
 984                 return fpr;
 985             } else {
 986                 terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 987                 return fprAllocate();
 988             }
 989         } else {
 990             DataFormat spillFormat = info.spillFormat();
 991             ASSERT((spillFormat & DataFormatJS) || spillFormat == DataFormatInteger);
 992             if (spillFormat == DataFormatJSDouble || spillFormat == DataFormatDouble) {
 993                 FPRReg fpr = fprAllocate();
 994                 m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr);
 995                 m_fprs.retain(fpr, virtualRegister, SpillOrderSpilled);
 996                 info.fillDouble(*m_stream, fpr);
 997                 return fpr;
 998             }
 999
1000             FPRReg fpr = fprAllocate();
1001             JITCompiler::Jump hasUnboxedDouble;
1002
1003             if (spillFormat != DataFormatJSInteger && spillFormat != DataFormatInteger) {
1004                 JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag));
1005                 if (type & ~SpecNumber)
1006                     speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::AboveOrEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::LowestTag)));
1007                 m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr);
1008                 hasUnboxedDouble = m_jit.jump();
1009
1010                 isInteger.link(&m_jit);
1011             }
1012
1013             m_jit.convertInt32ToDouble(JITCompiler::payloadFor(virtualRegister), fpr);
1014
1015             if (hasUnboxedDouble.isSet())
1016                 hasUnboxedDouble.link(&m_jit);
1017
1018             m_fprs.retain(fpr, virtualRegister, SpillOrderSpilled);
1019             info.fillDouble(*m_stream, fpr);
1020             info.killSpilled();
1021             return fpr;
1022         }
1023     }
1024
1025     switch (info.registerFormat()) {
1026     case DataFormatJS:
1027     case DataFormatJSInteger: {
1028         GPRReg tagGPR = info.tagGPR();
1029         GPRReg payloadGPR = info.payloadGPR();
1030         FPRReg fpr = fprAllocate();
1031
1032         m_gprs.lock(tagGPR);
1033         m_gprs.lock(payloadGPR);
1034
1035         JITCompiler::Jump hasUnboxedDouble;
1036
1037         if (info.registerFormat() != DataFormatJSInteger) {
1038             FPRTemporary scratch(this);
1039             JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag));
1040             if (type & ~SpecNumber)
1041                 speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::AboveOrEqual, tagGPR, TrustedImm32(JSValue::LowestTag)));
1042             unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
1043             hasUnboxedDouble = m_jit.jump();
1044             isInteger.link(&m_jit);
1045         }
1046
1047         m_jit.convertInt32ToDouble(payloadGPR, fpr);
1048
1049         if (hasUnboxedDouble.isSet())
1050             hasUnboxedDouble.link(&m_jit);
1051
1052         m_gprs.release(tagGPR);
1053         m_gprs.release(payloadGPR);
1054         m_gprs.unlock(tagGPR);
1055         m_gprs.unlock(payloadGPR);
1056         m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
1057         info.fillDouble(*m_stream, fpr);
1058         info.killSpilled();
1059         return fpr;
1060     }
1061
1062     case DataFormatInteger: {
1063         FPRReg fpr = fprAllocate();
1064         GPRReg gpr = info.gpr();
1065         m_gprs.lock(gpr);
1066         m_jit.convertInt32ToDouble(gpr, fpr);
1067         m_gprs.unlock(gpr);
1068         return fpr;
1069     }
1070
1071     case DataFormatJSDouble:
1072     case DataFormatDouble: {
1073         FPRReg fpr = info.fpr();
1074         m_fprs.lock(fpr);
1075         return fpr;
1076     }
1077
1078     case DataFormatNone:
1079     case DataFormatStorage:
1080         RELEASE_ASSERT_NOT_REACHED();
1081
1082     case DataFormatCell:
1083     case DataFormatJSCell:
1084     case DataFormatBoolean:
1085     case DataFormatJSBoolean:
1086         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1087         return fprAllocate();
1088
1089     default:
1090         RELEASE_ASSERT_NOT_REACHED();
1091         return InvalidFPRReg;
1092     }
1093 }
1094
1095 GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge)
1096 {
1097 #if DFG_ENABLE(DEBUG_VERBOSE)
1098     dataLogF("SpecCell@%d   ", edge->index());
1099 #endif
1100     AbstractValue& value = m_state.forNode(edge);
1101     SpeculatedType type = value.m_type;
1102     ASSERT((edge.useKind() != KnownCellUse && edge.useKind() != KnownStringUse) || !(value.m_type & ~SpecCell));
1103     value.filter(SpecCell);
1104     VirtualRegister virtualRegister = edge->virtualRegister();
1105     GenerationInfo& info = m_generationInfo[virtualRegister];
1106
1107     switch (info.registerFormat()) {
1108     case DataFormatNone: {
1109
1110         if (edge->hasConstant()) {
1111             JSValue jsValue = valueOfJSConstant(edge.node());
1112             GPRReg gpr = allocate();
1113             if (jsValue.isCell()) {
1114                 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
1115                 m_jit.move(MacroAssembler::TrustedImmPtr(jsValue.asCell()), gpr);
1116                 info.fillCell(*m_stream, gpr);
1117                 return gpr;
1118             }
1119             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1120             return gpr;
1121         }
1122
1123         ASSERT((info.spillFormat() & DataFormatJS) || info.spillFormat() == DataFormatCell);
1124         if (type & ~SpecCell)
1125             speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1126         GPRReg gpr = allocate();
1127         m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr);
1128         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
1129         info.fillCell(*m_stream, gpr);
1130         return gpr;
1131     }
1132
1133     case DataFormatCell: {
1134         GPRReg gpr = info.gpr();
1135         m_gprs.lock(gpr);
1136         return gpr;
1137     }
1138
1139     case DataFormatJSCell:
1140     case DataFormatJS: {
1141         GPRReg tagGPR = info.tagGPR();
1142         GPRReg payloadGPR = info.payloadGPR();
1143         m_gprs.lock(tagGPR);
1144         m_gprs.lock(payloadGPR);
1145         if (type & ~SpecCell)
1146             speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::CellTag)));
1147         m_gprs.unlock(tagGPR);
1148         m_gprs.release(tagGPR);
1149         m_gprs.release(payloadGPR);
1150         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderCell);
1151         info.fillCell(*m_stream, payloadGPR);
1152         return payloadGPR;
1153     }
1154
1155     case DataFormatJSInteger:
1156     case DataFormatInteger:
1157     case DataFormatJSDouble:
1158     case DataFormatDouble:
1159     case DataFormatJSBoolean:
1160     case DataFormatBoolean:
1161         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1162         return allocate();
1163
1164     case DataFormatStorage:
1165         RELEASE_ASSERT_NOT_REACHED();
1166
1167     default:
1168         RELEASE_ASSERT_NOT_REACHED();
1169         return InvalidGPRReg;
1170     }
1171 }
1172
1173 GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge)
1174 {
1175 #if DFG_ENABLE(DEBUG_VERBOSE)
1176     dataLogF("SpecBool@%d   ", edge.node()->index());
1177 #endif
1178     AbstractValue& value = m_state.forNode(edge);
1179     SpeculatedType type = value.m_type;
1180     value.filter(SpecBoolean);
1181     VirtualRegister virtualRegister = edge->virtualRegister();
1182     GenerationInfo& info = m_generationInfo[virtualRegister];
1183
1184     switch (info.registerFormat()) {
1185     case DataFormatNone: {
1186         if (info.spillFormat() == DataFormatInteger || info.spillFormat() == DataFormatDouble) {
1187             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1188             return allocate();
1189         }
1190
1191         if (edge->hasConstant()) {
1192             JSValue jsValue = valueOfJSConstant(edge.node());
1193             GPRReg gpr = allocate();
1194             if (jsValue.isBoolean()) {
1195                 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
1196                 m_jit.move(MacroAssembler::TrustedImm32(jsValue.asBoolean()), gpr);
1197                 info.fillBoolean(*m_stream, gpr);
1198                 return gpr;
1199             }
1200             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1201             return gpr;
1202         }
1203
1204         ASSERT((info.spillFormat() & DataFormatJS) || info.spillFormat() == DataFormatBoolean);
1205
1206         if (type & ~SpecBoolean)
1207             speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1208
1209         GPRReg gpr = allocate();
1210         m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr);
1211         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
1212         info.fillBoolean(*m_stream, gpr);
1213         return gpr;
1214     }
1215
1216     case DataFormatBoolean: {
1217         GPRReg gpr = info.gpr();
1218         m_gprs.lock(gpr);
1219         return gpr;
1220     }
1221
1222     case DataFormatJSBoolean:
1223     case DataFormatJS: {
1224         GPRReg tagGPR = info.tagGPR();
1225         GPRReg payloadGPR = info.payloadGPR();
1226         m_gprs.lock(tagGPR);
1227         m_gprs.lock(payloadGPR);
1228         if (type & ~SpecBoolean)
1229             speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::BooleanTag)));
1230
1231         m_gprs.unlock(tagGPR);
1232         m_gprs.release(tagGPR);
1233         m_gprs.release(payloadGPR);
1234         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderBoolean);
1235         info.fillBoolean(*m_stream, payloadGPR);
1236         return payloadGPR;
1237     }
1238
1239     case DataFormatJSInteger:
1240     case DataFormatInteger:
1241     case DataFormatJSDouble:
1242     case DataFormatDouble:
1243     case DataFormatJSCell:
1244     case DataFormatCell:
1245         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1246         return allocate();
1247
1248     case DataFormatStorage:
1249         RELEASE_ASSERT_NOT_REACHED();
1250
1251     default:
1252         RELEASE_ASSERT_NOT_REACHED();
1253         return InvalidGPRReg;
1254     }
1255 }
1256
1257 JITCompiler::Jump SpeculativeJIT::convertToDouble(JSValueOperand& op, FPRReg result)
1258 {
1259     FPRTemporary scratch(this);
1260
1261     GPRReg opPayloadGPR = op.payloadGPR();
1262     GPRReg opTagGPR = op.tagGPR();
1263     FPRReg scratchFPR = scratch.fpr();
1264
1265     JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, opTagGPR, TrustedImm32(JSValue::Int32Tag));
1266     JITCompiler::Jump notNumber = m_jit.branch32(MacroAssembler::AboveOrEqual, opPayloadGPR, TrustedImm32(JSValue::LowestTag));
1267
1268     unboxDouble(opTagGPR, opPayloadGPR, result, scratchFPR);
1269     JITCompiler::Jump done = m_jit.jump();
1270
1271     isInteger.link(&m_jit);
1272     m_jit.convertInt32ToDouble(opPayloadGPR, result);
1273
1274     done.link(&m_jit);
1275
1276     return notNumber;
1277 }
1278
1279 void SpeculativeJIT::compileObjectEquality(Node* node)
1280 {
1281     SpeculateCellOperand op1(this, node->child1());
1282     SpeculateCellOperand op2(this, node->child2());
1283     GPRReg op1GPR = op1.gpr();
1284     GPRReg op2GPR = op2.gpr();
1285
1286     if (m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
1287         m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1288         DFG_TYPE_CHECK(
1289             JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr(
1290                 MacroAssembler::Equal,
1291                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1292                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1293         DFG_TYPE_CHECK(
1294             JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr(
1295                 MacroAssembler::Equal,
1296                 MacroAssembler::Address(op2GPR, JSCell::structureOffset()),
1297                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1298     } else {
1299         GPRTemporary structure(this);
1300         GPRReg structureGPR = structure.gpr();
1301
1302         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1303         DFG_TYPE_CHECK(
1304             JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr(
1305                 MacroAssembler::Equal,
1306                 structureGPR,
1307                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1308         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1309             m_jit.branchTest8(
1310                 MacroAssembler::NonZero,
1311                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1312                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1313
1314         m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR);
1315         DFG_TYPE_CHECK(
1316             JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr(
1317                 MacroAssembler::Equal,
1318                 structureGPR,
1319                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1320         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1321             m_jit.branchTest8(
1322                 MacroAssembler::NonZero,
1323                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1324                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1325     }
1326
1327     GPRTemporary resultPayload(this, op2);
1328     GPRReg resultPayloadGPR = resultPayload.gpr();
1329
1330     MacroAssembler::Jump falseCase = m_jit.branchPtr(MacroAssembler::NotEqual, op1GPR, op2GPR);
1331     m_jit.move(TrustedImm32(1), resultPayloadGPR);
1332     MacroAssembler::Jump done = m_jit.jump();
1333     falseCase.link(&m_jit);
1334     m_jit.move(TrustedImm32(0), resultPayloadGPR);
1335     done.link(&m_jit);
1336
1337     booleanResult(resultPayloadGPR, node);
1338 }
1339
1340 void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild)
1341 {
1342     SpeculateCellOperand op1(this, leftChild);
1343     JSValueOperand op2(this, rightChild, ManualOperandSpeculation);
1344     GPRTemporary result(this);
1345
1346     GPRReg op1GPR = op1.gpr();
1347     GPRReg op2TagGPR = op2.tagGPR();
1348     GPRReg op2PayloadGPR = op2.payloadGPR();
1349     GPRReg resultGPR = result.gpr();
1350     GPRTemporary structure;
1351     GPRReg structureGPR = InvalidGPRReg;
1352
1353     bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid();
1354
1355     if (!masqueradesAsUndefinedWatchpointValid) {
1356         // The masquerades as undefined case will use the structure register, so allocate it here.
1357         // Do this at the top of the function to avoid branching around a register allocation.
1358         GPRTemporary realStructure(this);
1359         structure.adopt(realStructure);
1360         structureGPR = structure.gpr();
1361     }
1362
1363     if (masqueradesAsUndefinedWatchpointValid) {
1364         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1365         DFG_TYPE_CHECK(
1366             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1367                 MacroAssembler::Equal,
1368                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1369                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1370     } else {
1371         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1372         DFG_TYPE_CHECK(
1373             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1374                 MacroAssembler::Equal,
1375                 structureGPR,
1376                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1377         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild,
1378             m_jit.branchTest8(
1379                 MacroAssembler::NonZero,
1380                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1381                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1382     }
1383
1384
1385     // It seems that most of the time when programs do a == b where b may be either null/undefined
1386     // or an object, b is usually an object. Balance the branches to make that case fast.
1387     MacroAssembler::Jump rightNotCell =
1388         m_jit.branch32(MacroAssembler::NotEqual, op2TagGPR, TrustedImm32(JSValue::CellTag));
1389
1390     // We know that within this branch, rightChild must be a cell.
1391     if (masqueradesAsUndefinedWatchpointValid) {
1392         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1393         DFG_TYPE_CHECK(
1394             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject,
1395             m_jit.branchPtr(
1396                 MacroAssembler::Equal,
1397                 MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()),
1398                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1399     } else {
1400         m_jit.loadPtr(MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()), structureGPR);
1401         DFG_TYPE_CHECK(
1402             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject,
1403             m_jit.branchPtr(
1404                 MacroAssembler::Equal,
1405                 structureGPR,
1406                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1407         speculationCheck(BadType, JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild,
1408             m_jit.branchTest8(
1409                 MacroAssembler::NonZero,
1410                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1411                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1412     }
1413
1414     // At this point we know that we can perform a straight-forward equality comparison on pointer
1415     // values because both left and right are pointers to objects that have no special equality
1416     // protocols.
1417     MacroAssembler::Jump falseCase = m_jit.branchPtr(MacroAssembler::NotEqual, op1GPR, op2PayloadGPR);
1418     MacroAssembler::Jump trueCase = m_jit.jump();
1419
1420     rightNotCell.link(&m_jit);
1421
1422     // We know that within this branch, rightChild must not be a cell. Check if that is enough to
1423     // prove that it is either null or undefined.
1424     if (needsTypeCheck(rightChild, SpecCell | SpecOther)) {
1425         m_jit.move(op2TagGPR, resultGPR);
1426         m_jit.or32(TrustedImm32(1), resultGPR);
1427
1428         typeCheck(
1429             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, SpecCell | SpecOther,
1430             m_jit.branch32(
1431                 MacroAssembler::NotEqual, resultGPR,
1432                 MacroAssembler::TrustedImm32(JSValue::NullTag)));
1433     }
1434
1435     falseCase.link(&m_jit);
1436     m_jit.move(TrustedImm32(0), resultGPR);
1437     MacroAssembler::Jump done = m_jit.jump();
1438     trueCase.link(&m_jit);
1439     m_jit.move(TrustedImm32(1), resultGPR);
1440     done.link(&m_jit);
1441
1442     booleanResult(resultGPR, m_currentNode);
1443 }
1444
1445 void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild, Node* branchNode)
1446 {
1447     BlockIndex taken = branchNode->takenBlockIndex();
1448     BlockIndex notTaken = branchNode->notTakenBlockIndex();
1449
1450     SpeculateCellOperand op1(this, leftChild);
1451     JSValueOperand op2(this, rightChild, ManualOperandSpeculation);
1452     GPRTemporary result(this);
1453
1454     GPRReg op1GPR = op1.gpr();
1455     GPRReg op2TagGPR = op2.tagGPR();
1456     GPRReg op2PayloadGPR = op2.payloadGPR();
1457     GPRReg resultGPR = result.gpr();
1458     GPRTemporary structure;
1459     GPRReg structureGPR = InvalidGPRReg;
1460
1461     bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid();
1462
1463     if (!masqueradesAsUndefinedWatchpointValid) {
1464         // The masquerades as undefined case will use the structure register, so allocate it here.
1465         // Do this at the top of the function to avoid branching around a register allocation.
1466         GPRTemporary realStructure(this);
1467         structure.adopt(realStructure);
1468         structureGPR = structure.gpr();
1469     }
1470
1471     if (masqueradesAsUndefinedWatchpointValid) {
1472         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1473         DFG_TYPE_CHECK(
1474             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1475                 MacroAssembler::Equal,
1476                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1477                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1478     } else {
1479         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1480         DFG_TYPE_CHECK(
1481             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1482                 MacroAssembler::Equal,
1483                 structureGPR,
1484                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1485         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild,
1486             m_jit.branchTest8(
1487                 MacroAssembler::NonZero,
1488                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1489                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1490     }
1491
1492     // It seems that most of the time when programs do a == b where b may be either null/undefined
1493     // or an object, b is usually an object. Balance the branches to make that case fast.
1494     MacroAssembler::Jump rightNotCell =
1495         m_jit.branch32(MacroAssembler::NotEqual, op2TagGPR, TrustedImm32(JSValue::CellTag));
1496
1497     // We know that within this branch, rightChild must be a cell.
1498     if (masqueradesAsUndefinedWatchpointValid) {
1499         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1500         DFG_TYPE_CHECK(
1501             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject,
1502             m_jit.branchPtr(
1503                 MacroAssembler::Equal,
1504                 MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()),
1505                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1506     } else {
1507         m_jit.loadPtr(MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()), structureGPR);
1508         DFG_TYPE_CHECK(
1509             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject,
1510             m_jit.branchPtr(
1511                 MacroAssembler::Equal,
1512                 structureGPR,
1513                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1514         speculationCheck(BadType, JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild,
1515             m_jit.branchTest8(
1516                 MacroAssembler::NonZero,
1517                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1518                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1519     }
1520
1521     // At this point we know that we can perform a straight-forward equality comparison on pointer
1522     // values because both left and right are pointers to objects that have no special equality
1523     // protocols.
1524     branch32(MacroAssembler::Equal, op1GPR, op2PayloadGPR, taken);
1525
1526     // We know that within this branch, rightChild must not be a cell. Check if that is enough to
1527     // prove that it is either null or undefined.
1528     if (!needsTypeCheck(rightChild, SpecCell | SpecOther))
1529         rightNotCell.link(&m_jit);
1530     else {
1531         jump(notTaken, ForceJump);
1532
1533         rightNotCell.link(&m_jit);
1534         m_jit.move(op2TagGPR, resultGPR);
1535         m_jit.or32(TrustedImm32(1), resultGPR);
1536
1537         typeCheck(
1538             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, SpecCell | SpecOther,
1539             m_jit.branch32(
1540                 MacroAssembler::NotEqual, resultGPR,
1541                 MacroAssembler::TrustedImm32(JSValue::NullTag)));
1542     }
1543
1544     jump(notTaken);
1545 }
1546
1547 void SpeculativeJIT::compileIntegerCompare(Node* node, MacroAssembler::RelationalCondition condition)
1548 {
1549     SpeculateIntegerOperand op1(this, node->child1());
1550     SpeculateIntegerOperand op2(this, node->child2());
1551     GPRTemporary resultPayload(this);
1552
1553     m_jit.compare32(condition, op1.gpr(), op2.gpr(), resultPayload.gpr());
1554
1555     // If we add a DataFormatBool, we should use it here.
1556     booleanResult(resultPayload.gpr(), node);
1557 }
1558
1559 void SpeculativeJIT::compileDoubleCompare(Node* node, MacroAssembler::DoubleCondition condition)
1560 {
1561     SpeculateDoubleOperand op1(this, node->child1());
1562     SpeculateDoubleOperand op2(this, node->child2());
1563     GPRTemporary resultPayload(this);
1564
1565     m_jit.move(TrustedImm32(1), resultPayload.gpr());
1566     MacroAssembler::Jump trueCase = m_jit.branchDouble(condition, op1.fpr(), op2.fpr());
1567     m_jit.move(TrustedImm32(0), resultPayload.gpr());
1568     trueCase.link(&m_jit);
1569
1570     booleanResult(resultPayload.gpr(), node);
1571 }
1572
1573 void SpeculativeJIT::compileValueAdd(Node* node)
1574 {
1575     JSValueOperand op1(this, node->child1());
1576     JSValueOperand op2(this, node->child2());
1577
1578     GPRReg op1TagGPR = op1.tagGPR();
1579     GPRReg op1PayloadGPR = op1.payloadGPR();
1580     GPRReg op2TagGPR = op2.tagGPR();
1581     GPRReg op2PayloadGPR = op2.payloadGPR();
1582
1583     flushRegisters();
1584
1585     GPRResult2 resultTag(this);
1586     GPRResult resultPayload(this);
1587     if (isKnownNotNumber(node->child1().node()) || isKnownNotNumber(node->child2().node()))
1588         callOperation(operationValueAddNotNumber, resultTag.gpr(), resultPayload.gpr(), op1TagGPR, op1PayloadGPR, op2TagGPR, op2PayloadGPR);
1589     else
1590         callOperation(operationValueAdd, resultTag.gpr(), resultPayload.gpr(), op1TagGPR, op1PayloadGPR, op2TagGPR, op2PayloadGPR);
1591
1592     jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
1593 }
1594
1595 void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse)
1596 {
1597     JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
1598     GPRTemporary resultPayload(this);
1599     GPRReg valueTagGPR = value.tagGPR();
1600     GPRReg valuePayloadGPR = value.payloadGPR();
1601     GPRReg resultPayloadGPR = resultPayload.gpr();
1602     GPRTemporary structure;
1603     GPRReg structureGPR = InvalidGPRReg;
1604
1605     bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid();
1606
1607     if (!masqueradesAsUndefinedWatchpointValid) {
1608         // The masquerades as undefined case will use the structure register, so allocate it here.
1609         // Do this at the top of the function to avoid branching around a register allocation.
1610         GPRTemporary realStructure(this);
1611         structure.adopt(realStructure);
1612         structureGPR = structure.gpr();
1613     }
1614
1615     MacroAssembler::Jump notCell = m_jit.branch32(MacroAssembler::NotEqual, valueTagGPR, TrustedImm32(JSValue::CellTag));
1616     if (masqueradesAsUndefinedWatchpointValid) {
1617         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1618
1619         DFG_TYPE_CHECK(
1620             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject,
1621             m_jit.branchPtr(
1622                 MacroAssembler::Equal,
1623                 MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()),
1624                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1625     } else {
1626         m_jit.loadPtr(MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()), structureGPR);
1627
1628         DFG_TYPE_CHECK(
1629             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject,
1630             m_jit.branchPtr(
1631                 MacroAssembler::Equal,
1632                 structureGPR,
1633                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1634
1635         MacroAssembler::Jump isNotMasqueradesAsUndefined =
1636             m_jit.branchTest8(
1637                 MacroAssembler::Zero,
1638                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1639                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined));
1640
1641         speculationCheck(BadType, JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse,
1642             m_jit.branchPtr(
1643                 MacroAssembler::Equal,
1644                 MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()),
1645                 MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin))));
1646
1647         isNotMasqueradesAsUndefined.link(&m_jit);
1648     }
1649     m_jit.move(TrustedImm32(0), resultPayloadGPR);
1650     MacroAssembler::Jump done = m_jit.jump();
1651
1652     notCell.link(&m_jit);
1653
1654     COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag);
1655     if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) {
1656         m_jit.move(valueTagGPR, resultPayloadGPR);
1657         m_jit.or32(TrustedImm32(1), resultPayloadGPR);
1658         typeCheck(
1659             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, SpecCell | SpecOther,
1660             m_jit.branch32(
1661                 MacroAssembler::NotEqual,
1662                 resultPayloadGPR,
1663                 TrustedImm32(JSValue::NullTag)));
1664     }
1665     m_jit.move(TrustedImm32(1), resultPayloadGPR);
1666
1667     done.link(&m_jit);
1668
1669     booleanResult(resultPayloadGPR, m_currentNode);
1670 }
1671
1672 void SpeculativeJIT::compileLogicalNot(Node* node)
1673 {
1674     switch (node->child1().useKind()) {
1675     case BooleanUse: {
1676         SpeculateBooleanOperand value(this, node->child1());
1677         GPRTemporary result(this, value);
1678         m_jit.xor32(TrustedImm32(1), value.gpr(), result.gpr());
1679         booleanResult(result.gpr(), node);
1680         return;
1681     }
1682
1683     case ObjectOrOtherUse: {
1684         compileObjectOrOtherLogicalNot(node->child1());
1685         return;
1686     }
1687
1688     case Int32Use: {
1689         SpeculateIntegerOperand value(this, node->child1());
1690         GPRTemporary resultPayload(this, value);
1691         m_jit.compare32(MacroAssembler::Equal, value.gpr(), MacroAssembler::TrustedImm32(0), resultPayload.gpr());
1692         booleanResult(resultPayload.gpr(), node);
1693         return;
1694     }
1695
1696     case NumberUse: {
1697         SpeculateDoubleOperand value(this, node->child1());
1698         FPRTemporary scratch(this);
1699         GPRTemporary resultPayload(this);
1700         m_jit.move(TrustedImm32(0), resultPayload.gpr());
1701         MacroAssembler::Jump nonZero = m_jit.branchDoubleNonZero(value.fpr(), scratch.fpr());
1702         m_jit.move(TrustedImm32(1), resultPayload.gpr());
1703         nonZero.link(&m_jit);
1704         booleanResult(resultPayload.gpr(), node);
1705         return;
1706     }
1707
1708     case UntypedUse: {
1709         JSValueOperand arg1(this, node->child1());
1710         GPRTemporary resultPayload(this, arg1, false);
1711         GPRReg arg1TagGPR = arg1.tagGPR();
1712         GPRReg arg1PayloadGPR = arg1.payloadGPR();
1713         GPRReg resultPayloadGPR = resultPayload.gpr();
1714
1715         arg1.use();
1716
1717         JITCompiler::Jump slowCase = m_jit.branch32(JITCompiler::NotEqual, arg1TagGPR, TrustedImm32(JSValue::BooleanTag));
1718
1719         m_jit.move(arg1PayloadGPR, resultPayloadGPR);
1720
1721         addSlowPathGenerator(
1722             slowPathCall(
1723                 slowCase, this, dfgConvertJSValueToBoolean, resultPayloadGPR, arg1TagGPR,
1724                 arg1PayloadGPR));
1725
1726         m_jit.xor32(TrustedImm32(1), resultPayloadGPR);
1727         booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly);
1728         return;
1729     }
1730
1731     default:
1732         RELEASE_ASSERT_NOT_REACHED();
1733         break;
1734     }
1735 }
1736
1737 void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BlockIndex taken, BlockIndex notTaken)
1738 {
1739     JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
1740     GPRTemporary scratch(this);
1741     GPRReg valueTagGPR = value.tagGPR();
1742     GPRReg valuePayloadGPR = value.payloadGPR();
1743     GPRReg scratchGPR = scratch.gpr();
1744
1745     MacroAssembler::Jump notCell = m_jit.branch32(MacroAssembler::NotEqual, valueTagGPR, TrustedImm32(JSValue::CellTag));
1746     if (m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
1747         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1748
1749         DFG_TYPE_CHECK(
1750             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject,
1751             m_jit.branchPtr(
1752                 MacroAssembler::Equal,
1753                 MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()),
1754                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1755     } else {
1756         m_jit.loadPtr(MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()), scratchGPR);
1757
1758         DFG_TYPE_CHECK(
1759             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject,
1760             m_jit.branchPtr(
1761                 MacroAssembler::Equal,
1762                 scratchGPR,
1763                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1764
1765         JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::Zero, MacroAssembler::Address(scratchGPR, Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined));
1766
1767         speculationCheck(BadType, JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse,
1768             m_jit.branchPtr(
1769                 MacroAssembler::Equal,
1770                 MacroAssembler::Address(scratchGPR, Structure::globalObjectOffset()),
1771                 MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin))));
1772
1773         isNotMasqueradesAsUndefined.link(&m_jit);
1774     }
1775     jump(taken, ForceJump);
1776
1777     notCell.link(&m_jit);
1778
1779     COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag);
1780     if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) {
1781         m_jit.move(valueTagGPR, scratchGPR);
1782         m_jit.or32(TrustedImm32(1), scratchGPR);
1783         typeCheck(
1784             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, SpecCell | SpecOther,
1785             m_jit.branch32(MacroAssembler::NotEqual, scratchGPR, TrustedImm32(JSValue::NullTag)));
1786     }
1787
1788     jump(notTaken);
1789
1790     noResult(m_currentNode);
1791 }
1792
1793 void SpeculativeJIT::emitBranch(Node* node)
1794 {
1795     BlockIndex taken = node->takenBlockIndex();
1796     BlockIndex notTaken = node->notTakenBlockIndex();
1797
1798     switch (node->child1().useKind()) {
1799     case BooleanUse: {
1800         SpeculateBooleanOperand value(this, node->child1());
1801         MacroAssembler::ResultCondition condition = MacroAssembler::NonZero;
1802
1803         if (taken == nextBlock()) {
1804             condition = MacroAssembler::Zero;
1805             BlockIndex tmp = taken;
1806             taken = notTaken;
1807             notTaken = tmp;
1808         }
1809
1810         branchTest32(condition, value.gpr(), TrustedImm32(1), taken);
1811         jump(notTaken);
1812
1813         noResult(node);
1814         return;
1815     }
1816
1817     case ObjectOrOtherUse: {
1818         emitObjectOrOtherBranch(node->child1(), taken, notTaken);
1819         return;
1820     }
1821
1822     case NumberUse:
1823     case Int32Use: {
1824         if (node->child1().useKind() == Int32Use) {
1825             bool invert = false;
1826
1827             if (taken == nextBlock()) {
1828                 invert = true;
1829                 BlockIndex tmp = taken;
1830                 taken = notTaken;
1831                 notTaken = tmp;
1832             }
1833
1834             SpeculateIntegerOperand value(this, node->child1());
1835             branchTest32(invert ? MacroAssembler::Zero : MacroAssembler::NonZero, value.gpr(), taken);
1836         } else {
1837             SpeculateDoubleOperand value(this, node->child1());
1838             FPRTemporary scratch(this);
1839             branchDoubleNonZero(value.fpr(), scratch.fpr(), taken);
1840         }
1841
1842         jump(notTaken);
1843
1844         noResult(node);
1845         return;
1846     }
1847
1848     case UntypedUse: {
1849         JSValueOperand value(this, node->child1());
1850         value.fill();
1851         GPRReg valueTagGPR = value.tagGPR();
1852         GPRReg valuePayloadGPR = value.payloadGPR();
1853
1854         GPRTemporary result(this);
1855         GPRReg resultGPR = result.gpr();
1856
1857         use(node->child1());
1858
1859         JITCompiler::Jump fastPath = m_jit.branch32(JITCompiler::Equal, valueTagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag));
1860         JITCompiler::Jump slowPath = m_jit.branch32(JITCompiler::NotEqual, valueTagGPR, JITCompiler::TrustedImm32(JSValue::BooleanTag));
1861
1862         fastPath.link(&m_jit);
1863         branchTest32(JITCompiler::Zero, valuePayloadGPR, notTaken);
1864         jump(taken, ForceJump);
1865
1866         slowPath.link(&m_jit);
1867         silentSpillAllRegisters(resultGPR);
1868         callOperation(dfgConvertJSValueToBoolean, resultGPR, valueTagGPR, valuePayloadGPR);
1869         silentFillAllRegisters(resultGPR);
1870
1871         branchTest32(JITCompiler::NonZero, resultGPR, taken);
1872         jump(notTaken);
1873
1874         noResult(node, UseChildrenCalledExplicitly);
1875         return;
1876     }
1877
1878     default:
1879         RELEASE_ASSERT_NOT_REACHED();
1880         break;
1881     }
1882 }
1883
1884 template<typename BaseOperandType, typename PropertyOperandType, typename ValueOperandType, typename TagType>
1885 void SpeculativeJIT::compileContiguousPutByVal(Node* node, BaseOperandType& base, PropertyOperandType& property, ValueOperandType& value, GPRReg valuePayloadReg, TagType valueTag)
1886 {
1887     Edge child4 = m_jit.graph().varArgChild(node, 3);
1888
1889     ArrayMode arrayMode = node->arrayMode();
1890
1891     GPRReg baseReg = base.gpr();
1892     GPRReg propertyReg = property.gpr();
1893
1894     StorageOperand storage(this, child4);
1895     GPRReg storageReg = storage.gpr();
1896
1897     if (node->op() == PutByValAlias) {
1898         // Store the value to the array.
1899         GPRReg propertyReg = property.gpr();
1900         m_jit.store32(valueTag, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
1901         m_jit.store32(valuePayloadReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
1902
1903         noResult(node);
1904         return;
1905     }
1906
1907     MacroAssembler::Jump slowCase;
1908
1909     if (arrayMode.isInBounds()) {
1910         speculationCheck(
1911             StoreToHoleOrOutOfBounds, JSValueRegs(), 0,
1912             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
1913     } else {
1914         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
1915
1916         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
1917
1918         if (!arrayMode.isOutOfBounds())
1919             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
1920
1921         m_jit.add32(TrustedImm32(1), propertyReg);
1922         m_jit.store32(propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
1923         m_jit.sub32(TrustedImm32(1), propertyReg);
1924
1925         inBounds.link(&m_jit);
1926     }
1927
1928     m_jit.store32(valueTag, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
1929     m_jit.store32(valuePayloadReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
1930
1931     base.use();
1932     property.use();
1933     value.use();
1934     storage.use();
1935
1936     if (arrayMode.isOutOfBounds()) {
1937         addSlowPathGenerator(
1938             slowPathCall(
1939                 slowCase, this,
1940                 m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict,
1941                 NoResult, baseReg, propertyReg, valueTag, valuePayloadReg));
1942     }
1943
1944     noResult(node, UseChildrenCalledExplicitly);
1945 }
1946
1947 void SpeculativeJIT::compile(Node* node)
1948 {
1949     NodeType op = node->op();
1950
1951 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1952     m_jit.clearRegisterAllocationOffsets();
1953 #endif
1954
1955     switch (op) {
1956     case JSConstant:
1957         initConstantInfo(node);
1958         break;
1959
1960     case PhantomArguments:
1961         initConstantInfo(node);
1962         break;
1963
1964     case WeakJSConstant:
1965         m_jit.addWeakReference(node->weakConstant());
1966         initConstantInfo(node);
1967         break;
1968
1969     case Identity: {
1970         RELEASE_ASSERT_NOT_REACHED();
1971         break;
1972     }
1973
1974     case GetLocal: {
1975         SpeculatedType prediction = node->variableAccessData()->prediction();
1976         AbstractValue& value = m_state.variables().operand(node->local());
1977
1978         // If we have no prediction for this local, then don't attempt to compile.
1979         if (prediction == SpecNone) {
1980             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
1981             break;
1982         }
1983
1984         // If the CFA is tracking this variable and it found that the variable
1985         // cannot have been assigned, then don't attempt to proceed.
1986         if (value.isClear()) {
1987             // FIXME: We should trap instead.
1988             // https://bugs.webkit.org/show_bug.cgi?id=110383
1989             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
1990             break;
1991         }
1992
1993         if (node->variableAccessData()->shouldUseDoubleFormat()) {
1994             FPRTemporary result(this);
1995             m_jit.loadDouble(JITCompiler::addressFor(node->local()), result.fpr());
1996             VirtualRegister virtualRegister = node->virtualRegister();
1997             m_fprs.retain(result.fpr(), virtualRegister, SpillOrderDouble);
1998             m_generationInfo[virtualRegister].initDouble(node, node->refCount(), result.fpr());
1999             break;
2000         }
2001
2002         if (isInt32Speculation(value.m_type)) {
2003             GPRTemporary result(this);
2004             m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
2005
2006             // Like integerResult, but don't useChildren - our children are phi nodes,
2007             // and don't represent values within this dataflow with virtual registers.
2008             VirtualRegister virtualRegister = node->virtualRegister();
2009             m_gprs.retain(result.gpr(), virtualRegister, SpillOrderInteger);
2010             m_generationInfo[virtualRegister].initInteger(node, node->refCount(), result.gpr());
2011             break;
2012         }
2013
2014         if (isCellSpeculation(value.m_type)) {
2015             GPRTemporary result(this);
2016             m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
2017
2018             // Like cellResult, but don't useChildren - our children are phi nodes,
2019             // and don't represent values within this dataflow with virtual registers.
2020             VirtualRegister virtualRegister = node->virtualRegister();
2021             m_gprs.retain(result.gpr(), virtualRegister, SpillOrderCell);
2022             m_generationInfo[virtualRegister].initCell(node, node->refCount(), result.gpr());
2023             break;
2024         }
2025
2026         if (isBooleanSpeculation(value.m_type)) {
2027             GPRTemporary result(this);
2028             m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
2029
2030             // Like booleanResult, but don't useChildren - our children are phi nodes,
2031             // and don't represent values within this dataflow with virtual registers.
2032             VirtualRegister virtualRegister = node->virtualRegister();
2033             m_gprs.retain(result.gpr(), virtualRegister, SpillOrderBoolean);
2034             m_generationInfo[virtualRegister].initBoolean(node, node->refCount(), result.gpr());
2035             break;
2036         }
2037
2038         GPRTemporary result(this);
2039         GPRTemporary tag(this);
2040         m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
2041         m_jit.load32(JITCompiler::tagFor(node->local()), tag.gpr());
2042
2043         // Like jsValueResult, but don't useChildren - our children are phi nodes,
2044         // and don't represent values within this dataflow with virtual registers.
2045         VirtualRegister virtualRegister = node->virtualRegister();
2046         m_gprs.retain(result.gpr(), virtualRegister, SpillOrderJS);
2047         m_gprs.retain(tag.gpr(), virtualRegister, SpillOrderJS);
2048
2049         m_generationInfo[virtualRegister].initJSValue(node, node->refCount(), tag.gpr(), result.gpr(), DataFormatJS);
2050         break;
2051     }
2052
2053     case GetLocalUnlinked: {
2054         GPRTemporary payload(this);
2055         GPRTemporary tag(this);
2056         m_jit.load32(JITCompiler::payloadFor(node->unlinkedLocal()), payload.gpr());
2057         m_jit.load32(JITCompiler::tagFor(node->unlinkedLocal()), tag.gpr());
2058         jsValueResult(tag.gpr(), payload.gpr(), node);
2059         break;
2060     }
2061
2062     case MovHintAndCheck: {
2063         compileMovHintAndCheck(node);
2064         break;
2065     }
2066
2067     case InlineStart: {
2068         compileInlineStart(node);
2069         break;
2070     }
2071
2072     case MovHint:
2073     case ZombieHint: {
2074         RELEASE_ASSERT_NOT_REACHED();
2075         break;
2076     }
2077
2078     case SetLocal: {
2079         // SetLocal doubles as a hint as to where a node will be stored and
2080         // as a speculation point. So before we speculate make sure that we
2081         // know where the child of this node needs to go in the virtual
2082         // stack.
2083         compileMovHint(node);
2084
2085         if (node->variableAccessData()->shouldUnboxIfPossible()) {
2086             if (node->variableAccessData()->shouldUseDoubleFormat()) {
2087                 SpeculateDoubleOperand value(this, node->child1());
2088                 m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->local()));
2089                 noResult(node);
2090                 // Indicate that it's no longer necessary to retrieve the value of
2091                 // this bytecode variable from registers or other locations in the stack,
2092                 // but that it is stored as a double.
2093                 recordSetLocal(node->local(), ValueSource(DoubleInJSStack));
2094                 break;
2095             }
2096             SpeculatedType predictedType = node->variableAccessData()->argumentAwarePrediction();
2097             if (m_generationInfo[node->child1()->virtualRegister()].registerFormat() == DataFormatDouble) {
2098                 SpeculateDoubleOperand value(this, node->child1(), ManualOperandSpeculation);
2099                 m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->local()));
2100                 noResult(node);
2101                 recordSetLocal(node->local(), ValueSource(DoubleInJSStack));
2102                 break;
2103             }
2104             if (isInt32Speculation(predictedType)) {
2105                 SpeculateIntegerOperand value(this, node->child1());
2106                 m_jit.store32(value.gpr(), JITCompiler::payloadFor(node->local()));
2107                 noResult(node);
2108                 recordSetLocal(node->local(), ValueSource(Int32InJSStack));
2109                 break;
2110             }
2111             if (isCellSpeculation(predictedType)) {
2112                 SpeculateCellOperand cell(this, node->child1());
2113                 GPRReg cellGPR = cell.gpr();
2114                 m_jit.storePtr(cellGPR, JITCompiler::payloadFor(node->local()));
2115                 noResult(node);
2116                 recordSetLocal(node->local(), ValueSource(CellInJSStack));
2117                 break;
2118             }
2119             if (isBooleanSpeculation(predictedType)) {
2120                 SpeculateBooleanOperand value(this, node->child1());
2121                 m_jit.store32(value.gpr(), JITCompiler::payloadFor(node->local()));
2122                 noResult(node);
2123                 recordSetLocal(node->local(), ValueSource(BooleanInJSStack));
2124                 break;
2125             }
2126         }
2127         JSValueOperand value(this, node->child1());
2128         m_jit.store32(value.payloadGPR(), JITCompiler::payloadFor(node->local()));
2129         m_jit.store32(value.tagGPR(), JITCompiler::tagFor(node->local()));
2130         noResult(node);
2131         recordSetLocal(node->local(), ValueSource(ValueInJSStack));
2132
2133         // If we're storing an arguments object that has been optimized away,
2134         // our variable event stream for OSR exit now reflects the optimized
2135         // value (JSValue()). On the slow path, we want an arguments object
2136         // instead. We add an additional move hint to show OSR exit that it
2137         // needs to reconstruct the arguments object.
2138         if (node->child1()->op() == PhantomArguments)
2139             compileMovHint(node);
2140
2141         break;
2142     }
2143
2144     case SetArgument:
2145         // This is a no-op; it just marks the fact that the argument is being used.
2146         // But it may be profitable to use this as a hook to run speculation checks
2147         // on arguments, thereby allowing us to trivially eliminate such checks if
2148         // the argument is not used.
2149         break;
2150
2151     case BitAnd:
2152     case BitOr:
2153     case BitXor:
2154         if (isInt32Constant(node->child1().node())) {
2155             SpeculateIntegerOperand op2(this, node->child2());
2156             GPRTemporary result(this, op2);
2157
2158             bitOp(op, valueOfInt32Constant(node->child1().node()), op2.gpr(), result.gpr());
2159
2160             integerResult(result.gpr(), node);
2161         } else if (isInt32Constant(node->child2().node())) {
2162             SpeculateIntegerOperand op1(this, node->child1());
2163             GPRTemporary result(this, op1);
2164
2165             bitOp(op, valueOfInt32Constant(node->child2().node()), op1.gpr(), result.gpr());
2166
2167             integerResult(result.gpr(), node);
2168         } else {
2169             SpeculateIntegerOperand op1(this, node->child1());
2170             SpeculateIntegerOperand op2(this, node->child2());
2171             GPRTemporary result(this, op1, op2);
2172
2173             GPRReg reg1 = op1.gpr();
2174             GPRReg reg2 = op2.gpr();
2175             bitOp(op, reg1, reg2, result.gpr());
2176
2177             integerResult(result.gpr(), node);
2178         }
2179         break;
2180
2181     case BitRShift:
2182     case BitLShift:
2183     case BitURShift:
2184         if (isInt32Constant(node->child2().node())) {
2185             SpeculateIntegerOperand op1(this, node->child1());
2186             GPRTemporary result(this, op1);
2187
2188             shiftOp(op, op1.gpr(), valueOfInt32Constant(node->child2().node()) & 0x1f, result.gpr());
2189
2190             integerResult(result.gpr(), node);
2191         } else {
2192             // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
2193             SpeculateIntegerOperand op1(this, node->child1());
2194             SpeculateIntegerOperand op2(this, node->child2());
2195             GPRTemporary result(this, op1);
2196
2197             GPRReg reg1 = op1.gpr();
2198             GPRReg reg2 = op2.gpr();
2199             shiftOp(op, reg1, reg2, result.gpr());
2200
2201             integerResult(result.gpr(), node);
2202         }
2203         break;
2204
2205     case UInt32ToNumber: {
2206         compileUInt32ToNumber(node);
2207         break;
2208     }
2209
2210     case DoubleAsInt32: {
2211         compileDoubleAsInt32(node);
2212         break;
2213     }
2214
2215     case ValueToInt32: {
2216         compileValueToInt32(node);
2217         break;
2218     }
2219
2220     case Int32ToDouble:
2221     case ForwardInt32ToDouble: {
2222         compileInt32ToDouble(node);
2223         break;
2224     }
2225
2226     case ValueAdd:
2227     case ArithAdd:
2228         compileAdd(node);
2229         break;
2230
2231     case MakeRope:
2232         compileMakeRope(node);
2233         break;
2234
2235     case ArithSub:
2236         compileArithSub(node);
2237         break;
2238
2239     case ArithNegate:
2240         compileArithNegate(node);
2241         break;
2242
2243     case ArithMul:
2244         compileArithMul(node);
2245         break;
2246
2247     case ArithIMul:
2248         compileArithIMul(node);
2249         break;
2250
2251     case ArithDiv: {
2252         switch (node->binaryUseKind()) {
2253         case Int32Use: {
2254 #if CPU(X86)
2255             compileIntegerArithDivForX86(node);
2256 #elif CPU(ARM64)
2257             compileIntegerArithDivForARM64(node);
2258 #elif CPU(APPLE_ARMV7S)
2259             compileIntegerArithDivForARMv7s(node);
2260 #else // CPU type without integer divide
2261             RELEASE_ASSERT_NOT_REACHED(); // should have been coverted into a double divide.
2262 #endif
2263             break;
2264         }
2265
2266         case NumberUse: {
2267             SpeculateDoubleOperand op1(this, node->child1());
2268             SpeculateDoubleOperand op2(this, node->child2());
2269             FPRTemporary result(this, op1);
2270
2271             FPRReg reg1 = op1.fpr();
2272             FPRReg reg2 = op2.fpr();
2273             m_jit.divDouble(reg1, reg2, result.fpr());
2274
2275             doubleResult(result.fpr(), node);
2276             break;
2277         }
2278
2279         default:
2280             RELEASE_ASSERT_NOT_REACHED();
2281             break;
2282         }
2283         break;
2284     }
2285
2286     case ArithMod: {
2287         compileArithMod(node);
2288         break;
2289     }
2290
2291     case ArithAbs: {
2292         switch (node->child1().useKind()) {
2293         case Int32Use: {
2294             SpeculateIntegerOperand op1(this, node->child1());
2295             GPRTemporary result(this, op1);
2296             GPRTemporary scratch(this);
2297
2298             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2299             m_jit.rshift32(result.gpr(), MacroAssembler::TrustedImm32(31), scratch.gpr());
2300             m_jit.add32(scratch.gpr(), result.gpr());
2301             m_jit.xor32(scratch.gpr(), result.gpr());
2302             speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, result.gpr(), MacroAssembler::TrustedImm32(1 << 31)));
2303             integerResult(result.gpr(), node);
2304             break;
2305         }
2306
2307
2308         case NumberUse: {
2309             SpeculateDoubleOperand op1(this, node->child1());
2310             FPRTemporary result(this);
2311
2312             m_jit.absDouble(op1.fpr(), result.fpr());
2313             doubleResult(result.fpr(), node);
2314             break;
2315         }
2316
2317         default:
2318             RELEASE_ASSERT_NOT_REACHED();
2319             break;
2320         }
2321         break;
2322     }
2323
2324     case ArithMin:
2325     case ArithMax: {
2326         switch (node->binaryUseKind()) {
2327         case Int32Use: {
2328             SpeculateStrictInt32Operand op1(this, node->child1());
2329             SpeculateStrictInt32Operand op2(this, node->child2());
2330             GPRTemporary result(this, op1);
2331
2332             GPRReg op1GPR = op1.gpr();
2333             GPRReg op2GPR = op2.gpr();
2334             GPRReg resultGPR = result.gpr();
2335
2336             MacroAssembler::Jump op1Less = m_jit.branch32(op == ArithMin ? MacroAssembler::LessThan : MacroAssembler::GreaterThan, op1GPR, op2GPR);
2337             m_jit.move(op2GPR, resultGPR);
2338             if (op1GPR != resultGPR) {
2339                 MacroAssembler::Jump done = m_jit.jump();
2340                 op1Less.link(&m_jit);
2341                 m_jit.move(op1GPR, resultGPR);
2342                 done.link(&m_jit);
2343             } else
2344                 op1Less.link(&m_jit);
2345
2346             integerResult(resultGPR, node);
2347             break;
2348         }
2349
2350         case NumberUse: {
2351             SpeculateDoubleOperand op1(this, node->child1());
2352             SpeculateDoubleOperand op2(this, node->child2());
2353             FPRTemporary result(this, op1);
2354
2355             FPRReg op1FPR = op1.fpr();
2356             FPRReg op2FPR = op2.fpr();
2357             FPRReg resultFPR = result.fpr();
2358
2359             MacroAssembler::JumpList done;
2360
2361             MacroAssembler::Jump op1Less = m_jit.branchDouble(op == ArithMin ? MacroAssembler::DoubleLessThan : MacroAssembler::DoubleGreaterThan, op1FPR, op2FPR);
2362
2363             // op2 is eather the lesser one or one of then is NaN
2364             MacroAssembler::Jump op2Less = m_jit.branchDouble(op == ArithMin ? MacroAssembler::DoubleGreaterThanOrEqual : MacroAssembler::DoubleLessThanOrEqual, op1FPR, op2FPR);
2365
2366             // Unordered case. We don't know which of op1, op2 is NaN. Manufacture NaN by adding
2367             // op1 + op2 and putting it into result.
2368             m_jit.addDouble(op1FPR, op2FPR, resultFPR);
2369             done.append(m_jit.jump());
2370
2371             op2Less.link(&m_jit);
2372             m_jit.moveDouble(op2FPR, resultFPR);
2373
2374             if (op1FPR != resultFPR) {
2375                 done.append(m_jit.jump());
2376
2377                 op1Less.link(&m_jit);
2378                 m_jit.moveDouble(op1FPR, resultFPR);
2379             } else
2380                 op1Less.link(&m_jit);
2381
2382             done.link(&m_jit);
2383
2384             doubleResult(resultFPR, node);
2385             break;
2386         }
2387
2388         default:
2389             RELEASE_ASSERT_NOT_REACHED();
2390             break;
2391         }
2392         break;
2393     }
2394
2395     case ArithSqrt: {
2396         SpeculateDoubleOperand op1(this, node->child1());
2397         FPRTemporary result(this, op1);
2398
2399         m_jit.sqrtDouble(op1.fpr(), result.fpr());
2400
2401         doubleResult(result.fpr(), node);
2402         break;
2403     }
2404
2405     case LogicalNot:
2406         compileLogicalNot(node);
2407         break;
2408
2409     case CompareLess:
2410         if (compare(node, JITCompiler::LessThan, JITCompiler::DoubleLessThan, operationCompareLess))
2411             return;
2412         break;
2413
2414     case CompareLessEq:
2415         if (compare(node, JITCompiler::LessThanOrEqual, JITCompiler::DoubleLessThanOrEqual, operationCompareLessEq))
2416             return;
2417         break;
2418
2419     case CompareGreater:
2420         if (compare(node, JITCompiler::GreaterThan, JITCompiler::DoubleGreaterThan, operationCompareGreater))
2421             return;
2422         break;
2423
2424     case CompareGreaterEq:
2425         if (compare(node, JITCompiler::GreaterThanOrEqual, JITCompiler::DoubleGreaterThanOrEqual, operationCompareGreaterEq))
2426             return;
2427         break;
2428
2429     case CompareEqConstant:
2430         ASSERT(isNullConstant(node->child2().node()));
2431         if (nonSpeculativeCompareNull(node, node->child1()))
2432             return;
2433         break;
2434
2435     case CompareEq:
2436         if (compare(node, JITCompiler::Equal, JITCompiler::DoubleEqual, operationCompareEq))
2437             return;
2438         break;
2439
2440     case CompareStrictEqConstant:
2441         if (compileStrictEqForConstant(node, node->child1(), valueOfJSConstant(node->child2().node())))
2442             return;
2443         break;
2444
2445     case CompareStrictEq:
2446         if (compileStrictEq(node))
2447             return;
2448         break;
2449
2450     case StringCharCodeAt: {
2451         compileGetCharCodeAt(node);
2452         break;
2453     }
2454
2455     case StringCharAt: {
2456         // Relies on StringCharAt node having same basic layout as GetByVal
2457         compileGetByValOnString(node);
2458         break;
2459     }
2460
2461     case StringFromCharCode: {
2462         compileFromCharCode(node);
2463         break;
2464     }
2465
2466     case CheckArray: {
2467         checkArray(node);
2468         break;
2469     }
2470
2471     case Arrayify:
2472     case ArrayifyToStructure: {
2473         arrayify(node);
2474         break;
2475     }
2476
2477     case GetByVal: {
2478         switch (node->arrayMode().type()) {
2479         case Array::SelectUsingPredictions:
2480         case Array::ForceExit:
2481             RELEASE_ASSERT_NOT_REACHED();
2482             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
2483             break;
2484         case Array::Generic: {
2485             SpeculateCellOperand base(this, node->child1()); // Save a register, speculate cell. We'll probably be right.
2486             JSValueOperand property(this, node->child2());
2487             GPRReg baseGPR = base.gpr();
2488             GPRReg propertyTagGPR = property.tagGPR();
2489             GPRReg propertyPayloadGPR = property.payloadGPR();
2490
2491             flushRegisters();
2492             GPRResult2 resultTag(this);
2493             GPRResult resultPayload(this);
2494             callOperation(operationGetByValCell, resultTag.gpr(), resultPayload.gpr(), baseGPR, propertyTagGPR, propertyPayloadGPR);
2495
2496             jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
2497             break;
2498         }
2499         case Array::Int32:
2500         case Array::Contiguous: {
2501             if (node->arrayMode().isInBounds()) {
2502                 SpeculateStrictInt32Operand property(this, node->child2());
2503                 StorageOperand storage(this, node->child3());
2504
2505                 GPRReg propertyReg = property.gpr();
2506                 GPRReg storageReg = storage.gpr();
2507
2508                 if (!m_compileOkay)
2509                     return;
2510
2511                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2512
2513                 GPRTemporary resultPayload(this);
2514                 if (node->arrayMode().type() == Array::Int32) {
2515                     speculationCheck(
2516                         OutOfBounds, JSValueRegs(), 0,
2517                         m_jit.branch32(
2518                             MacroAssembler::Equal,
2519                             MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)),
2520                             TrustedImm32(JSValue::EmptyValueTag)));
2521                     m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayload.gpr());
2522                     integerResult(resultPayload.gpr(), node);
2523                     break;
2524                 }
2525
2526                 GPRTemporary resultTag(this);
2527                 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTag.gpr());
2528                 speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, resultTag.gpr(), TrustedImm32(JSValue::EmptyValueTag)));
2529                 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayload.gpr());
2530                 jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
2531                 break;
2532             }
2533
2534             SpeculateCellOperand base(this, node->child1());
2535             SpeculateStrictInt32Operand property(this, node->child2());
2536             StorageOperand storage(this, node->child3());
2537
2538             GPRReg baseReg = base.gpr();
2539             GPRReg propertyReg = property.gpr();
2540             GPRReg storageReg = storage.gpr();
2541
2542             if (!m_compileOkay)
2543                 return;
2544
2545             GPRTemporary resultTag(this);
2546             GPRTemporary resultPayload(this);
2547             GPRReg resultTagReg = resultTag.gpr();
2548             GPRReg resultPayloadReg = resultPayload.gpr();
2549
2550             MacroAssembler::JumpList slowCases;
2551
2552             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2553
2554             m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTagReg);
2555             m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayloadReg);
2556             slowCases.append(m_jit.branch32(MacroAssembler::Equal, resultTagReg, TrustedImm32(JSValue::EmptyValueTag)));
2557
2558             addSlowPathGenerator(
2559                 slowPathCall(
2560                     slowCases, this, operationGetByValArrayInt,
2561                     JSValueRegs(resultTagReg, resultPayloadReg), baseReg, propertyReg));
2562
2563             jsValueResult(resultTagReg, resultPayloadReg, node);
2564             break;
2565         }
2566         case Array::Double: {
2567             if (node->arrayMode().isInBounds()) {
2568                 if (node->arrayMode().isSaneChain()) {
2569                     JSGlobalObject* globalObject = m_jit.globalObjectFor(node->codeOrigin);
2570                     ASSERT(globalObject->arrayPrototypeChainIsSane());
2571                     globalObject->arrayPrototype()->structure()->addTransitionWatchpoint(speculationWatchpoint());
2572                     globalObject->objectPrototype()->structure()->addTransitionWatchpoint(speculationWatchpoint());
2573                 }
2574
2575                 SpeculateStrictInt32Operand property(this, node->child2());
2576                 StorageOperand storage(this, node->child3());
2577
2578                 GPRReg propertyReg = property.gpr();
2579                 GPRReg storageReg = storage.gpr();
2580
2581                 if (!m_compileOkay)
2582                     return;
2583
2584                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2585
2586                 FPRTemporary result(this);
2587                 m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), result.fpr());
2588                 if (!node->arrayMode().isSaneChain())
2589                     speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, result.fpr(), result.fpr()));
2590                 doubleResult(result.fpr(), node);
2591                 break;
2592             }
2593
2594             SpeculateCellOperand base(this, node->child1());
2595             SpeculateStrictInt32Operand property(this, node->child2());
2596             StorageOperand storage(this, node->child3());
2597
2598             GPRReg baseReg = base.gpr();
2599             GPRReg propertyReg = property.gpr();
2600             GPRReg storageReg = storage.gpr();
2601
2602             if (!m_compileOkay)
2603                 return;
2604
2605             GPRTemporary resultTag(this);
2606             GPRTemporary resultPayload(this);
2607             FPRTemporary temp(this);
2608             GPRReg resultTagReg = resultTag.gpr();
2609             GPRReg resultPayloadReg = resultPayload.gpr();
2610             FPRReg tempReg = temp.fpr();
2611
2612             MacroAssembler::JumpList slowCases;
2613
2614             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2615
2616             m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), tempReg);
2617             slowCases.append(m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempReg, tempReg));
2618             boxDouble(tempReg, resultTagReg, resultPayloadReg);
2619
2620             addSlowPathGenerator(
2621                 slowPathCall(
2622                     slowCases, this, operationGetByValArrayInt,
2623                     JSValueRegs(resultTagReg, resultPayloadReg), baseReg, propertyReg));
2624
2625             jsValueResult(resultTagReg, resultPayloadReg, node);
2626             break;
2627         }
2628         case Array::ArrayStorage:
2629         case Array::SlowPutArrayStorage: {
2630             if (node->arrayMode().isInBounds()) {
2631                 SpeculateStrictInt32Operand property(this, node->child2());
2632                 StorageOperand storage(this, node->child3());
2633                 GPRReg propertyReg = property.gpr();
2634                 GPRReg storageReg = storage.gpr();
2635
2636                 if (!m_compileOkay)
2637                     return;
2638
2639                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset())));
2640
2641                 GPRTemporary resultTag(this);
2642                 GPRTemporary resultPayload(this);
2643
2644                 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTag.gpr());
2645                 speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, resultTag.gpr(), TrustedImm32(JSValue::EmptyValueTag)));
2646                 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayload.gpr());
2647
2648                 jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
2649                 break;
2650             }
2651
2652             SpeculateCellOperand base(this, node->child1());
2653             SpeculateStrictInt32Operand property(this, node->child2());
2654             StorageOperand storage(this, node->child3());
2655             GPRReg propertyReg = property.gpr();
2656             GPRReg storageReg = storage.gpr();
2657             GPRReg baseReg = base.gpr();
2658
2659             if (!m_compileOkay)
2660                 return;
2661
2662             GPRTemporary resultTag(this);
2663             GPRTemporary resultPayload(this);
2664             GPRReg resultTagReg = resultTag.gpr();
2665             GPRReg resultPayloadReg = resultPayload.gpr();
2666
2667             JITCompiler::Jump outOfBounds = m_jit.branch32(
2668                 MacroAssembler::AboveOrEqual, propertyReg,
2669                 MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset()));
2670
2671             m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTagReg);
2672             JITCompiler::Jump hole = m_jit.branch32(
2673                 MacroAssembler::Equal, resultTag.gpr(), TrustedImm32(JSValue::EmptyValueTag));
2674             m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayloadReg);
2675
2676             JITCompiler::JumpList slowCases;
2677             slowCases.append(outOfBounds);
2678             slowCases.append(hole);
2679             addSlowPathGenerator(
2680                 slowPathCall(
2681                     slowCases, this, operationGetByValArrayInt,
2682                     JSValueRegs(resultTagReg, resultPayloadReg),
2683                     baseReg, propertyReg));
2684
2685             jsValueResult(resultTagReg, resultPayloadReg, node);
2686             break;
2687         }
2688         case Array::String:
2689             compileGetByValOnString(node);
2690             break;
2691         case Array::Arguments:
2692             compileGetByValOnArguments(node);
2693             break;
2694         case Array::Int8Array:
2695             compileGetByValOnIntTypedArray(m_jit.vm()->int8ArrayDescriptor(), node, sizeof(int8_t), SignedTypedArray);
2696             break;
2697         case Array::Int16Array:
2698             compileGetByValOnIntTypedArray(m_jit.vm()->int16ArrayDescriptor(), node, sizeof(int16_t), SignedTypedArray);
2699             break;
2700         case Array::Int32Array:
2701             compileGetByValOnIntTypedArray(m_jit.vm()->int32ArrayDescriptor(), node, sizeof(int32_t), SignedTypedArray);
2702             break;
2703         case Array::Uint8Array:
2704             compileGetByValOnIntTypedArray(m_jit.vm()->uint8ArrayDescriptor(), node, sizeof(uint8_t), UnsignedTypedArray);
2705             break;
2706         case Array::Uint8ClampedArray:
2707             compileGetByValOnIntTypedArray(m_jit.vm()->uint8ClampedArrayDescriptor(), node, sizeof(uint8_t), UnsignedTypedArray);
2708             break;
2709         case Array::Uint16Array:
2710             compileGetByValOnIntTypedArray(m_jit.vm()->uint16ArrayDescriptor(), node, sizeof(uint16_t), UnsignedTypedArray);
2711             break;
2712         case Array::Uint32Array:
2713             compileGetByValOnIntTypedArray(m_jit.vm()->uint32ArrayDescriptor(), node, sizeof(uint32_t), UnsignedTypedArray);
2714             break;
2715         case Array::Float32Array:
2716             compileGetByValOnFloatTypedArray(m_jit.vm()->float32ArrayDescriptor(), node, sizeof(float));
2717             break;
2718         case Array::Float64Array:
2719             compileGetByValOnFloatTypedArray(m_jit.vm()->float64ArrayDescriptor(), node, sizeof(double));
2720             break;
2721         default:
2722             RELEASE_ASSERT_NOT_REACHED();
2723             break;
2724         }
2725         break;
2726     }
2727
2728     case PutByVal:
2729     case PutByValAlias: {
2730         Edge child1 = m_jit.graph().varArgChild(node, 0);
2731         Edge child2 = m_jit.graph().varArgChild(node, 1);
2732         Edge child3 = m_jit.graph().varArgChild(node, 2);
2733         Edge child4 = m_jit.graph().varArgChild(node, 3);
2734
2735         ArrayMode arrayMode = node->arrayMode().modeForPut();
2736         bool alreadyHandled = false;
2737
2738         switch (arrayMode.type()) {
2739         case Array::SelectUsingPredictions:
2740         case Array::ForceExit:
2741             RELEASE_ASSERT_NOT_REACHED();
2742             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
2743             alreadyHandled = true;
2744             break;
2745         case Array::Generic: {
2746             ASSERT(node->op() == PutByVal);
2747
2748             SpeculateCellOperand base(this, child1); // Save a register, speculate cell. We'll probably be right.
2749             JSValueOperand property(this, child2);
2750             JSValueOperand value(this, child3);
2751             GPRReg baseGPR = base.gpr();
2752             GPRReg propertyTagGPR = property.tagGPR();
2753             GPRReg propertyPayloadGPR = property.payloadGPR();
2754             GPRReg valueTagGPR = value.tagGPR();
2755             GPRReg valuePayloadGPR = value.payloadGPR();
2756
2757             flushRegisters();
2758             callOperation(m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict, baseGPR, propertyTagGPR, propertyPayloadGPR, valueTagGPR, valuePayloadGPR);
2759
2760             noResult(node);
2761             alreadyHandled = true;
2762             break;
2763         }
2764         default:
2765             break;
2766         }
2767
2768         if (alreadyHandled)
2769             break;
2770
2771         SpeculateCellOperand base(this, child1);
2772         SpeculateStrictInt32Operand property(this, child2);
2773
2774         GPRReg baseReg = base.gpr();
2775         GPRReg propertyReg = property.gpr();
2776
2777         switch (arrayMode.type()) {
2778         case Array::Int32: {
2779             SpeculateIntegerOperand value(this, child3);
2780
2781             GPRReg valuePayloadReg = value.gpr();
2782
2783             if (!m_compileOkay)
2784                 return;
2785
2786             compileContiguousPutByVal(node, base, property, value, valuePayloadReg, TrustedImm32(JSValue::Int32Tag));
2787             break;
2788         }
2789         case Array::Contiguous: {
2790             JSValueOperand value(this, child3);
2791
2792             GPRReg valueTagReg = value.tagGPR();
2793             GPRReg valuePayloadReg = value.payloadGPR();
2794
2795             if (!m_compileOkay)
2796                 return;
2797
2798             if (Heap::isWriteBarrierEnabled()) {
2799                 GPRTemporary scratch(this);
2800                 writeBarrier(baseReg, valueTagReg, child3, WriteBarrierForPropertyAccess, scratch.gpr());
2801             }
2802
2803             compileContiguousPutByVal(node, base, property, value, valuePayloadReg, valueTagReg);
2804             break;
2805         }
2806         case Array::Double: {
2807             compileDoublePutByVal(node, base, property);
2808             break;
2809         }
2810         case Array::ArrayStorage:
2811         case Array::SlowPutArrayStorage: {
2812             JSValueOperand value(this, child3);
2813
2814             GPRReg valueTagReg = value.tagGPR();
2815             GPRReg valuePayloadReg = value.payloadGPR();
2816
2817             if (!m_compileOkay)
2818                 return;
2819
2820             {
2821                 GPRTemporary scratch(this);
2822                 GPRReg scratchReg = scratch.gpr();
2823                 writeBarrier(baseReg, valueTagReg, child3, WriteBarrierForPropertyAccess, scratchReg);
2824             }
2825
2826             StorageOperand storage(this, child4);
2827             GPRReg storageReg = storage.gpr();
2828
2829             if (node->op() == PutByValAlias) {
2830                 // Store the value to the array.
2831                 GPRReg propertyReg = property.gpr();
2832                 m_jit.store32(value.tagGPR(), MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
2833                 m_jit.store32(value.payloadGPR(), MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
2834
2835                 noResult(node);
2836                 break;
2837             }
2838
2839             MacroAssembler::JumpList slowCases;
2840
2841             MacroAssembler::Jump beyondArrayBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset()));
2842             if (!arrayMode.isOutOfBounds())
2843                 speculationCheck(OutOfBounds, JSValueRegs(), 0, beyondArrayBounds);
2844             else
2845                 slowCases.append(beyondArrayBounds);
2846
2847             // Check if we're writing to a hole; if so increment m_numValuesInVector.
2848             if (arrayMode.isInBounds()) {
2849                 speculationCheck(
2850                     StoreToHole, JSValueRegs(), 0,
2851                     m_jit.branch32(MacroAssembler::Equal, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), TrustedImm32(JSValue::EmptyValueTag)));
2852             } else {
2853                 MacroAssembler::Jump notHoleValue = m_jit.branch32(MacroAssembler::NotEqual, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), TrustedImm32(JSValue::EmptyValueTag));
2854                 if (arrayMode.isSlowPut()) {
2855                     // This is sort of strange. If we wanted to optimize this code path, we would invert
2856                     // the above branch. But it's simply not worth it since this only happens if we're
2857                     // already having a bad time.
2858                     slowCases.append(m_jit.jump());
2859                 } else {
2860                     m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageReg, ArrayStorage::numValuesInVectorOffset()));
2861
2862                     // If we're writing to a hole we might be growing the array;
2863                     MacroAssembler::Jump lengthDoesNotNeedUpdate = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::lengthOffset()));
2864                     m_jit.add32(TrustedImm32(1), propertyReg);
2865                     m_jit.store32(propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::lengthOffset()));
2866                     m_jit.sub32(TrustedImm32(1), propertyReg);
2867
2868                     lengthDoesNotNeedUpdate.link(&m_jit);
2869                 }
2870                 notHoleValue.link(&m_jit);
2871             }
2872
2873             // Store the value to the array.
2874             m_jit.store32(valueTagReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
2875             m_jit.store32(valuePayloadReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
2876
2877             base.use();
2878             property.use();
2879             value.use();
2880             storage.use();
2881
2882             if (!slowCases.empty()) {
2883                 addSlowPathGenerator(
2884                     slowPathCall(
2885                         slowCases, this,
2886                         m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict,
2887                         NoResult, baseReg, propertyReg, valueTagReg, valuePayloadReg));
2888             }
2889
2890             noResult(node, UseChildrenCalledExplicitly);
2891             break;
2892         }
2893
2894         case Array::Arguments:
2895             // FIXME: we could at some point make this work. Right now we're assuming that the register
2896             // pressure would be too great.
2897             RELEASE_ASSERT_NOT_REACHED();
2898             break;
2899
2900         case Array::Int8Array:
2901             compilePutByValForIntTypedArray(m_jit.vm()->int8ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int8_t), SignedTypedArray);
2902             break;
2903
2904         case Array::Int16Array:
2905             compilePutByValForIntTypedArray(m_jit.vm()->int16ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int16_t), SignedTypedArray);
2906             break;
2907
2908         case Array::Int32Array:
2909             compilePutByValForIntTypedArray(m_jit.vm()->int32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int32_t), SignedTypedArray);
2910             break;
2911
2912         case Array::Uint8Array:
2913             compilePutByValForIntTypedArray(m_jit.vm()->uint8ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint8_t), UnsignedTypedArray);
2914             break;
2915
2916         case Array::Uint8ClampedArray:
2917             compilePutByValForIntTypedArray(m_jit.vm()->uint8ClampedArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint8_t), UnsignedTypedArray, ClampRounding);
2918             break;
2919
2920         case Array::Uint16Array:
2921             compilePutByValForIntTypedArray(m_jit.vm()->uint16ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint16_t), UnsignedTypedArray);
2922             break;
2923
2924         case Array::Uint32Array:
2925             compilePutByValForIntTypedArray(m_jit.vm()->uint32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint32_t), UnsignedTypedArray);
2926             break;
2927
2928         case Array::Float32Array:
2929             compilePutByValForFloatTypedArray(m_jit.vm()->float32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(float));
2930             break;
2931
2932         case Array::Float64Array:
2933             compilePutByValForFloatTypedArray(m_jit.vm()->float64ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(double));
2934             break;
2935
2936         default:
2937             RELEASE_ASSERT_NOT_REACHED();
2938             break;
2939         }
2940         break;
2941     }
2942
2943     case RegExpExec: {
2944         if (compileRegExpExec(node))
2945             return;
2946
2947         if (!node->adjustedRefCount()) {
2948             SpeculateCellOperand base(this, node->child1());
2949             SpeculateCellOperand argument(this, node->child2());
2950             GPRReg baseGPR = base.gpr();
2951             GPRReg argumentGPR = argument.gpr();
2952
2953             flushRegisters();
2954             GPRResult result(this);
2955             callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR);
2956
2957             // Must use jsValueResult because otherwise we screw up register
2958             // allocation, which thinks that this node has a result.
2959             booleanResult(result.gpr(), node);
2960             break;
2961         }
2962
2963         SpeculateCellOperand base(this, node->child1());
2964         SpeculateCellOperand argument(this, node->child2());
2965         GPRReg baseGPR = base.gpr();
2966         GPRReg argumentGPR = argument.gpr();
2967
2968         flushRegisters();
2969         GPRResult2 resultTag(this);
2970         GPRResult resultPayload(this);
2971         callOperation(operationRegExpExec, resultTag.gpr(), resultPayload.gpr(), baseGPR, argumentGPR);
2972
2973         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
2974         break;
2975     }
2976
2977     case RegExpTest: {
2978         SpeculateCellOperand base(this, node->child1());
2979         SpeculateCellOperand argument(this, node->child2());
2980         GPRReg baseGPR = base.gpr();
2981         GPRReg argumentGPR = argument.gpr();
2982
2983         flushRegisters();
2984         GPRResult result(this);
2985         callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR);
2986
2987         // If we add a DataFormatBool, we should use it here.
2988         booleanResult(result.gpr(), node);
2989         break;
2990     }
2991
2992     case ArrayPush: {
2993         ASSERT(node->arrayMode().isJSArray());
2994
2995         SpeculateCellOperand base(this, node->child1());
2996         GPRTemporary storageLength(this);
2997
2998         GPRReg baseGPR = base.gpr();
2999         GPRReg storageLengthGPR = storageLength.gpr();
3000
3001         StorageOperand storage(this, node->child3());
3002         GPRReg storageGPR = storage.gpr();
3003
3004         switch (node->arrayMode().type()) {
3005         case Array::Int32: {
3006             SpeculateIntegerOperand value(this, node->child2());
3007             GPRReg valuePayloadGPR = value.gpr();
3008
3009             m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
3010             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3011             m_jit.store32(TrustedImm32(JSValue::Int32Tag), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3012             m_jit.store32(valuePayloadGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3013             m_jit.add32(TrustedImm32(1), storageLengthGPR);
3014             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3015             m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR);
3016
3017             addSlowPathGenerator(
3018                 slowPathCall(
3019                     slowPath, this, operationArrayPush,
3020                     JSValueRegs(storageGPR, storageLengthGPR),
3021                     TrustedImm32(JSValue::Int32Tag), valuePayloadGPR, baseGPR));
3022
3023             jsValueResult(storageGPR, storageLengthGPR, node);
3024             break;
3025         }
3026
3027         case Array::Contiguous: {
3028             JSValueOperand value(this, node->child2());
3029             GPRReg valueTagGPR = value.tagGPR();
3030             GPRReg valuePayloadGPR = value.payloadGPR();
3031
3032             if (Heap::isWriteBarrierEnabled()) {
3033                 GPRTemporary scratch(this);
3034                 writeBarrier(baseGPR, valueTagGPR, node->child2(), WriteBarrierForPropertyAccess, scratch.gpr(), storageLengthGPR);
3035             }
3036
3037             m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
3038             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3039             m_jit.store32(valueTagGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3040             m_jit.store32(valuePayloadGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3041             m_jit.add32(TrustedImm32(1), storageLengthGPR);
3042             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3043             m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR);
3044
3045             addSlowPathGenerator(
3046                 slowPathCall(
3047                     slowPath, this, operationArrayPush,
3048                     JSValueRegs(storageGPR, storageLengthGPR),
3049                     valueTagGPR, valuePayloadGPR, baseGPR));
3050
3051             jsValueResult(storageGPR, storageLengthGPR, node);
3052             break;
3053         }
3054
3055         case Array::Double: {
3056             SpeculateDoubleOperand value(this, node->child2());
3057             FPRReg valueFPR = value.fpr();
3058
3059             DFG_TYPE_CHECK(
3060                 JSValueRegs(), node->child2(), SpecRealNumber,
3061                 m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, valueFPR, valueFPR));
3062
3063             m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
3064             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3065             m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
3066             m_jit.add32(TrustedImm32(1), storageLengthGPR);
3067             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3068             m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR);
3069
3070             addSlowPathGenerator(
3071                 slowPathCall(
3072                     slowPath, this, operationArrayPushDouble,
3073                     JSValueRegs(storageGPR, storageLengthGPR),
3074                     valueFPR, baseGPR));
3075
3076             jsValueResult(storageGPR, storageLengthGPR, node);
3077             break;
3078         }
3079
3080         case Array::ArrayStorage: {
3081             JSValueOperand value(this, node->child2());
3082             GPRReg valueTagGPR = value.tagGPR();
3083             GPRReg valuePayloadGPR = value.payloadGPR();
3084
3085             if (Heap::isWriteBarrierEnabled()) {
3086                 GPRTemporary scratch(this);
3087                 writeBarrier(baseGPR, valueTagGPR, node->child2(), WriteBarrierForPropertyAccess, scratch.gpr(), storageLengthGPR);
3088             }
3089
3090             m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
3091
3092             // Refuse to handle bizarre lengths.
3093             speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Above, storageLengthGPR, TrustedImm32(0x7ffffffe)));
3094
3095             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
3096
3097             m_jit.store32(valueTagGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3098             m_jit.store32(valuePayloadGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3099
3100             m_jit.add32(TrustedImm32(1), storageLengthGPR);
3101             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
3102             m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
3103             m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR);
3104
3105             addSlowPathGenerator(slowPathCall(slowPath, this, operationArrayPush, JSValueRegs(storageGPR, storageLengthGPR), valueTagGPR, valuePayloadGPR, baseGPR));
3106
3107             jsValueResult(storageGPR, storageLengthGPR, node);
3108             break;
3109         }
3110
3111         default:
3112             CRASH();
3113             break;
3114         }
3115         break;
3116     }
3117
3118     case ArrayPop: {
3119         ASSERT(node->arrayMode().isJSArray());
3120
3121         SpeculateCellOperand base(this, node->child1());
3122         StorageOperand storage(this, node->child2());
3123         GPRTemporary valueTag(this);
3124         GPRTemporary valuePayload(this);
3125
3126         GPRReg baseGPR = base.gpr();
3127         GPRReg valueTagGPR = valueTag.gpr();
3128         GPRReg valuePayloadGPR = valuePayload.gpr();
3129         GPRReg storageGPR = storage.gpr();
3130
3131         switch (node->arrayMode().type()) {
3132         case Array::Int32:
3133         case Array::Contiguous: {
3134             m_jit.load32(
3135                 MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), valuePayloadGPR);
3136             MacroAssembler::Jump undefinedCase =
3137                 m_jit.branchTest32(MacroAssembler::Zero, valuePayloadGPR);
3138             m_jit.sub32(TrustedImm32(1), valuePayloadGPR);
3139             m_jit.store32(
3140                 valuePayloadGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3141             m_jit.load32(
3142                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)),
3143                 valueTagGPR);
3144             MacroAssembler::Jump slowCase = m_jit.branch32(MacroAssembler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
3145             m_jit.store32(
3146                 MacroAssembler::TrustedImm32(JSValue::EmptyValueTag),
3147                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3148             m_jit.load32(
3149                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)),
3150                 valuePayloadGPR);
3151
3152             addSlowPathGenerator(
3153                 slowPathMove(
3154                     undefinedCase, this,
3155                     MacroAssembler::TrustedImm32(jsUndefined().tag()), valueTagGPR,
3156                     MacroAssembler::TrustedImm32(jsUndefined().payload()), valuePayloadGPR));
3157             addSlowPathGenerator(
3158                 slowPathCall(
3159                     slowCase, this, operationArrayPopAndRecoverLength,
3160                     JSValueRegs(valueTagGPR, valuePayloadGPR), baseGPR));
3161
3162             jsValueResult(valueTagGPR, valuePayloadGPR, node);
3163             break;
3164         }
3165
3166         case Array::Double: {
3167             FPRTemporary temp(this);
3168             FPRReg tempFPR = temp.fpr();
3169
3170             m_jit.load32(
3171                 MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), valuePayloadGPR);
3172             MacroAssembler::Jump undefinedCase =
3173                 m_jit.branchTest32(MacroAssembler::Zero, valuePayloadGPR);
3174             m_jit.sub32(TrustedImm32(1), valuePayloadGPR);
3175             m_jit.store32(
3176                 valuePayloadGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3177             m_jit.loadDouble(
3178                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight),
3179                 tempFPR);
3180             MacroAssembler::Jump slowCase = m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempFPR, tempFPR);
3181             JSValue nan = JSValue(JSValue::EncodeAsDouble, QNaN);
3182             m_jit.store32(
3183                 MacroAssembler::TrustedImm32(nan.u.asBits.tag),
3184                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3185             m_jit.store32(
3186                 MacroAssembler::TrustedImm32(nan.u.asBits.payload),
3187                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3188             boxDouble(tempFPR, valueTagGPR, valuePayloadGPR);
3189
3190             addSlowPathGenerator(
3191                 slowPathMove(
3192                     undefinedCase, this,
3193                     MacroAssembler::TrustedImm32(jsUndefined().tag()), valueTagGPR,
3194                     MacroAssembler::TrustedImm32(jsUndefined().payload()), valuePayloadGPR));
3195             addSlowPathGenerator(
3196                 slowPathCall(
3197                     slowCase, this, operationArrayPopAndRecoverLength,
3198                     JSValueRegs(valueTagGPR, valuePayloadGPR), baseGPR));
3199
3200             jsValueResult(valueTagGPR, valuePayloadGPR, node);
3201             break;
3202         }
3203
3204         case Array::ArrayStorage: {
3205             GPRTemporary storageLength(this);
3206             GPRReg storageLengthGPR = storageLength.gpr();
3207
3208             m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
3209
3210             JITCompiler::JumpList setUndefinedCases;
3211             setUndefinedCases.append(m_jit.branchTest32(MacroAssembler::Zero, storageLengthGPR));
3212
3213             m_jit.sub32(TrustedImm32(1), storageLengthGPR);
3214
3215             MacroAssembler::Jump slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
3216
3217             m_jit.load32(MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), valueTagGPR);
3218             m_jit.load32(MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), valuePayloadGPR);
3219
3220             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
3221
3222             setUndefinedCases.append(m_jit.branch32(MacroAssembler::Equal, TrustedImm32(JSValue::EmptyValueTag), valueTagGPR));
3223
3224             m_jit.store32(TrustedImm32(JSValue::EmptyValueTag), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3225
3226             m_jit.sub32(TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
3227
3228             addSlowPathGenerator(
3229                 slowPathMove(
3230                     setUndefinedCases, this,
3231                     MacroAssembler::TrustedImm32(jsUndefined().tag()), valueTagGPR,
3232                     MacroAssembler::TrustedImm32(jsUndefined().payload()), valuePayloadGPR));
3233
3234             addSlowPathGenerator(
3235                 slowPathCall(
3236                     slowCase, this, operationArrayPop,
3237                     JSValueRegs(valueTagGPR, valuePayloadGPR), baseGPR));
3238
3239             jsValueResult(valueTagGPR, valuePayloadGPR, node);
3240             break;
3241         }
3242
3243         default:
3244             CRASH();
3245             break;
3246         }
3247         break;
3248     }
3249
3250     case DFG::Jump: {
3251         BlockIndex taken = node->takenBlockIndex();
3252         jump(taken);
3253         noResult(node);
3254         break;
3255     }
3256
3257     case Branch:
3258         emitBranch(node);
3259         break;
3260
3261     case Return: {
3262         ASSERT(GPRInfo::callFrameRegister != GPRInfo::regT2);
3263         ASSERT(GPRInfo::regT1 != GPRInfo::returnValueGPR);
3264         ASSERT(GPRInfo::returnValueGPR != GPRInfo::callFrameRegister);
3265
3266 #if DFG_ENABLE(SUCCESS_STATS)
3267         static SamplingCounter counter("SpeculativeJIT");
3268         m_jit.emitCount(counter);
3269 #endif
3270
3271         // Return the result in returnValueGPR.
3272         JSValueOperand op1(this, node->child1());
3273         op1.fill();
3274         if (op1.isDouble())
3275             boxDouble(op1.fpr(), GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
3276         else {
3277             if (op1.payloadGPR() == GPRInfo::returnValueGPR2 && op1.tagGPR() == GPRInfo::returnValueGPR)
3278                 m_jit.swap(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR2);
3279             else if (op1.payloadGPR() == GPRInfo::returnValueGPR2) {
3280                 m_jit.move(op1.payloadGPR(), GPRInfo::returnValueGPR);
3281                 m_jit.move(op1.tagGPR(), GPRInfo::returnValueGPR2);
3282             } else {
3283                 m_jit.move(op1.tagGPR(), GPRInfo::returnValueGPR2);
3284                 m_jit.move(op1.payloadGPR(), GPRInfo::returnValueGPR);
3285             }
3286         }
3287
3288         // Grab the return address.
3289         m_jit.emitGetFromCallFrameHeaderPtr(JSStack::ReturnPC, GPRInfo::regT2);
3290         // Restore our caller's "r".
3291         m_jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, GPRInfo::callFrameRegister);
3292         // Return.
3293         m_jit.restoreReturnAddressBeforeReturn(GPRInfo::regT2);
3294         m_jit.ret();
3295
3296         noResult(node);
3297         break;
3298     }
3299
3300     case Throw:
3301     case ThrowReferenceError: {
3302         // We expect that throw statements are rare and are intended to exit the code block
3303         // anyway, so we just OSR back to the old JIT for now.
3304         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
3305         break;
3306     }
3307
3308     case ToPrimitive: {
3309         RELEASE_ASSERT(node->child1().useKind() == UntypedUse);
3310         JSValueOperand op1(this, node->child1());
3311         GPRTemporary resultTag(this, op1);
3312         GPRTemporary resultPayload(this, op1, false);
3313
3314         GPRReg op1TagGPR = op1.tagGPR();
3315         GPRReg op1PayloadGPR = op1.payloadGPR();
3316         GPRReg resultTagGPR = resultTag.gpr();
3317         GPRReg resultPayloadGPR = resultPayload.gpr();
3318
3319         op1.use();
3320
3321         if (!(m_state.forNode(node->child1()).m_type & ~(SpecNumber | SpecBoolean))) {
3322             m_jit.move(op1TagGPR, resultTagGPR);
3323             m_jit.move(op1PayloadGPR, resultPayloadGPR);
3324         } else {
3325             MacroAssembler::Jump alreadyPrimitive = m_jit.branch32(MacroAssembler::NotEqual, op1TagGPR, TrustedImm32(JSValue::CellTag));
3326             MacroAssembler::Jump notPrimitive = m_jit.branchPtr(MacroAssembler::NotEqual, MacroAssembler::Address(op1PayloadGPR, JSCell::structureOffset()), MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()));
3327
3328             alreadyPrimitive.link(&m_jit);
3329             m_jit.move(op1TagGPR, resultTagGPR);
3330             m_jit.move(op1PayloadGPR, resultPayloadGPR);
3331
3332             addSlowPathGenerator(
3333                 slowPathCall(
3334                     notPrimitive, this, operationToPrimitive,
3335                     JSValueRegs(resultTagGPR, resultPayloadGPR), op1TagGPR, op1PayloadGPR));
3336         }
3337
3338         jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3339         break;
3340     }
3341
3342     case ToString: {
3343         if (node->child1().useKind() == UntypedUse) {
3344             JSValueOperand op1(this, node->child1());
3345             GPRReg op1PayloadGPR = op1.payloadGPR();
3346             GPRReg op1TagGPR = op1.tagGPR();
3347
3348             GPRResult result(this);
3349             GPRReg resultGPR = result.gpr();
3350
3351             flushRegisters();
3352
3353             JITCompiler::Jump done;
3354             if (node->child1()->prediction() & SpecString) {
3355                 JITCompiler::Jump slowPath1 = m_jit.branch32(
3356                     JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::CellTag));
3357                 JITCompiler::Jump slowPath2 = m_jit.branchPtr(
3358                     JITCompiler::NotEqual,
3359                     JITCompiler::Address(op1PayloadGPR, JSCell::structureOffset()),
3360                     TrustedImmPtr(m_jit.vm()->stringStructure.get()));
3361                 m_jit.move(op1PayloadGPR, resultGPR);
3362                 done = m_jit.jump();
3363                 slowPath1.link(&m_jit);
3364                 slowPath2.link(&m_jit);
3365             }
3366             callOperation(operationToString, resultGPR, op1TagGPR, op1PayloadGPR);
3367             if (done.isSet())
3368                 done.link(&m_jit);
3369             cellResult(resultGPR, node);
3370             break;
3371         }
3372
3373         compileToStringOnCell(node);
3374         break;
3375     }
3376
3377     case NewStringObject: {
3378         compileNewStringObject(node);
3379         break;
3380     }
3381
3382     case NewArray: {
3383         JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin);
3384         if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) {
3385             globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint());
3386
3387             Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType());
3388             ASSERT(structure->indexingType() == node->indexingType());
3389             ASSERT(
3390                 hasUndecided(structure->indexingType())
3391                 || hasInt32(structure->indexingType())
3392                 || hasDouble(structure->indexingType())
3393                 || hasContiguous(structure->indexingType()));
3394
3395             unsigned numElements = node->numChildren();
3396
3397             GPRTemporary result(this);
3398             GPRTemporary storage(this);
3399
3400             GPRReg resultGPR = result.gpr();
3401             GPRReg storageGPR = storage.gpr();
3402
3403             emitAllocateJSArray(resultGPR, structure, storageGPR, numElements);
3404
3405             // At this point, one way or another, resultGPR and storageGPR have pointers to
3406             // the JSArray and the Butterfly, respectively.
3407
3408             ASSERT(!hasUndecided(structure->indexingType()) || !node->numChildren());
3409
3410             for (unsigned operandIdx = 0; operandIdx < node->numChildren(); ++operandIdx) {
3411                 Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx];
3412                 switch (node->indexingType()) {
3413                 case ALL_BLANK_INDEXING_TYPES:
3414                 case ALL_UNDECIDED_INDEXING_TYPES:
3415                     CRASH();
3416                     break;
3417                 case ALL_DOUBLE_INDEXING_TYPES: {
3418                     SpeculateDoubleOperand operand(this, use);
3419                     FPRReg opFPR = operand.fpr();
3420                     DFG_TYPE_CHECK(
3421                         JSValueRegs(), use, SpecRealNumber,
3422                         m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR));
3423
3424                     m_jit.storeDouble(opFPR, MacroAssembler::Address(storageGPR, sizeof(double) * operandIdx));
3425                     break;
3426                 }
3427                 case ALL_INT32_INDEXING_TYPES: {
3428                     SpeculateIntegerOperand operand(this, use);
3429                     m_jit.store32(TrustedImm32(JSValue::Int32Tag), MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3430                     m_jit.store32(operand.gpr(), MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3431                     break;
3432                 }
3433                 case ALL_CONTIGUOUS_INDEXING_TYPES: {
3434                     JSValueOperand operand(this, m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx]);
3435                     GPRReg opTagGPR = operand.tagGPR();
3436                     GPRReg opPayloadGPR = operand.payloadGPR();
3437                     m_jit.store32(opTagGPR, MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3438                     m_jit.store32(opPayloadGPR, MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3439                     break;
3440                 }
3441                 default:
3442                     CRASH();
3443                     break;
3444                 }
3445             }
3446
3447             // Yuck, we should *really* have a way of also returning the storageGPR. But
3448             // that's the least of what's wrong with this code. We really shouldn't be
3449             // allocating the array after having computed - and probably spilled to the
3450             // stack - all of the things that will go into the array. The solution to that
3451             // bigger problem will also likely fix the redundancy in reloading the storage
3452             // pointer that we currently have.
3453
3454             cellResult(resultGPR, node);
3455             break;
3456         }
3457
3458         if (!node->numChildren()) {
3459             flushRegisters();
3460             GPRResult result(this);
3461             callOperation(
3462                 operationNewEmptyArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()));
3463             cellResult(result.gpr(), node);
3464             break;
3465         }
3466
3467         size_t scratchSize = sizeof(EncodedJSValue) * node->numChildren();
3468         ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(scratchSize);
3469         EncodedJSValue* buffer = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
3470
3471         for (unsigned operandIdx = 0; operandIdx < node->numChildren(); ++operandIdx) {
3472             // Need to perform the speculations that this node promises to perform. If we're
3473             // emitting code here and the indexing type is not array storage then there is
3474             // probably something hilarious going on and we're already failing at all the
3475             // things, but at least we're going to be sound.
3476             Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx];
3477             switch (node->indexingType()) {
3478             case ALL_BLANK_INDEXING_TYPES:
3479             case ALL_UNDECIDED_INDEXING_TYPES:
3480                 CRASH();
3481                 break;
3482             case ALL_DOUBLE_INDEXING_TYPES: {
3483                 SpeculateDoubleOperand operand(this, use);
3484                 FPRReg opFPR = operand.fpr();
3485                 DFG_TYPE_CHECK(
3486                     JSValueRegs(), use, SpecRealNumber,
3487                     m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR));
3488
3489                 m_jit.storeDouble(opFPR, reinterpret_cast<char*>(buffer + operandIdx));
3490                 break;
3491             }
3492             case ALL_INT32_INDEXING_TYPES: {
3493                 SpeculateIntegerOperand operand(this, use);
3494                 GPRReg opGPR = operand.gpr();
3495                 m_jit.store32(TrustedImm32(JSValue::Int32Tag), reinterpret_cast<char*>(buffer + operandIdx) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
3496                 m_jit.store32(opGPR, reinterpret_cast<char*>(buffer + operandIdx) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
3497                 break;
3498             }
3499             case ALL_CONTIGUOUS_INDEXING_TYPES:
3500             case ALL_ARRAY_STORAGE_INDEXING_TYPES: {
3501                 JSValueOperand operand(this, m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx]);
3502                 GPRReg opTagGPR = operand.tagGPR();
3503                 GPRReg opPayloadGPR = operand.payloadGPR();
3504
3505                 m_jit.store32(opTagGPR, reinterpret_cast<char*>(buffer + operandIdx) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
3506                 m_jit.store32(opPayloadGPR, reinterpret_cast<char*>(buffer + operandIdx) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
3507                 operand.use();
3508                 break;
3509             }
3510             default:
3511                 CRASH();
3512                 break;
3513             }
3514         }
3515
3516         switch (node->indexingType()) {
3517         case ALL_DOUBLE_INDEXING_TYPES:
3518         case ALL_INT32_INDEXING_TYPES:
3519             useChildren(node);
3520             break;
3521         default:
3522             break;
3523         }
3524
3525         flushRegisters();
3526
3527         if (scratchSize) {
3528             GPRTemporary scratch(this);
3529
3530             // Tell GC mark phase how much of the scratch buffer is active during call.
3531             m_jit.move(TrustedImmPtr(scratchBuffer->activeLengthPtr()), scratch.gpr());
3532             m_jit.storePtr(TrustedImmPtr(scratchSize), scratch.gpr());
3533         }
3534
3535         GPRResult result(this);
3536
3537         callOperation(
3538             operationNewArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()),
3539             static_cast<void*>(buffer), node->numChildren());
3540
3541         if (scratchSize) {
3542             GPRTemporary scratch(this);
3543
3544             m_jit.move(TrustedImmPtr(scratchBuffer->activeLengthPtr()), scratch.gpr());
3545             m_jit.storePtr(TrustedImmPtr(0), scratch.gpr());
3546         }
3547
3548         cellResult(result.gpr(), node, UseChildrenCalledExplicitly);
3549         break;
3550     }
3551
3552     case NewArrayWithSize: {
3553         JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin);
3554         if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) {
3555             globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint());
3556
3557             SpeculateStrictInt32Operand size(this, node->child1());
3558             GPRTemporary result(this);
3559             GPRTemporary storage(this);
3560             GPRTemporary scratch(this);
3561             GPRTemporary scratch2(this);
3562
3563             GPRReg sizeGPR = size.gpr();
3564             GPRReg resultGPR = result.gpr();
3565             GPRReg storageGPR = storage.gpr();
3566             GPRReg scratchGPR = scratch.gpr();
3567             GPRReg scratch2GPR = scratch2.gpr();
3568
3569             MacroAssembler::JumpList slowCases;
3570             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX)));
3571
3572             ASSERT((1 << 3) == sizeof(JSValue));
3573             m_jit.move(sizeGPR, scratchGPR);
3574             m_jit.lshift32(TrustedImm32(3), scratchGPR);
3575             m_jit.add32(TrustedImm32(sizeof(IndexingHeader)), scratchGPR, resultGPR);
3576             slowCases.append(
3577                 emitAllocateBasicStorage(resultGPR, storageGPR));
3578             m_jit.subPtr(scratchGPR, storageGPR);
3579             Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType());
3580             emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases);
3581
3582             m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3583             m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3584
3585             if (hasDouble(node->indexingType())) {
3586                 JSValue nan = JSValue(JSValue::EncodeAsDouble, QNaN);
3587
3588                 m_jit.move(sizeGPR, scratchGPR);
3589                 MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratchGPR);
3590                 MacroAssembler::Label loop = m_jit.label();
3591                 m_jit.sub32(TrustedImm32(1), scratchGPR);
3592                 m_jit.store32(TrustedImm32(nan.u.asBits.tag), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3593                 m_jit.store32(TrustedImm32(nan.u.asBits.payload), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3594                 m_jit.branchTest32(MacroAssembler::NonZero, scratchGPR).linkTo(loop, &m_jit);
3595                 done.link(&m_jit);
3596             }
3597
3598             addSlowPathGenerator(adoptPtr(
3599                 new CallArrayAllocatorWithVariableSizeSlowPathGenerator(
3600                     slowCases, this, operationNewArrayWithSize, resultGPR,
3601                     globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()),
3602                     globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage),
3603                     sizeGPR)));
3604
3605             cellResult(resultGPR, node);
3606             break;
3607         }
3608
3609         SpeculateStrictInt32Operand size(this, node->child1());
3610         GPRReg sizeGPR = size.gpr();
3611         flushRegisters();
3612         GPRResult result(this);
3613         GPRReg resultGPR = result.gpr();
3614         GPRReg structureGPR = selectScratchGPR(sizeGPR);
3615         MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX));
3616         m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), structureGPR);
3617         MacroAssembler::Jump done = m_jit.jump();
3618         bigLength.link(&m_jit);
3619         m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)), structureGPR);
3620         done.link(&m_jit);
3621         callOperation(
3622             operationNewArrayWithSize, resultGPR, structureGPR, sizeGPR);
3623         cellResult(resultGPR, node);
3624         break;
3625     }
3626
3627     case NewArrayBuffer: {
3628         JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin);
3629         IndexingType indexingType = node->indexingType();
3630         if (!globalObject->isHavingABadTime() && !hasArrayStorage(indexingType)) {
3631             globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint());
3632
3633             unsigned numElements = node->numConstants();
3634
3635             GPRTemporary result(this);
3636             GPRTemporary storage(this);
3637
3638             GPRReg resultGPR = result.gpr();
3639             GPRReg storageGPR = storage.gpr();
3640
3641             emitAllocateJSArray(resultGPR, globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType), storageGPR, numElements);
3642
3643             if (node->indexingType() == ArrayWithDouble) {
3644                 JSValue* data = m_jit.codeBlock()->constantBuffer(node->startConstant());
3645                 for (unsigned index = 0; index < node->numConstants(); ++index) {
3646                     union {
3647                         int32_t halves[2];
3648                         double value;
3649                     } u;
3650                     u.value = data[index].asNumber();
3651                     m_jit.store32(Imm32(u.halves[0]), MacroAssembler::Address(storageGPR, sizeof(double) * index));
3652                     m_jit.store32(Imm32(u.halves[1]), MacroAssembler::Address(storageGPR, sizeof(double) * index + sizeof(int32_t)));
3653                 }
3654             } else {
3655                 int32_t* data = bitwise_cast<int32_t*>(m_jit.codeBlock()->constantBuffer(node->startConstant()));
3656                 for (unsigned index = 0; index < node->numConstants() * 2; ++index) {
3657                     m_jit.store32(
3658                         Imm32(data[index]), MacroAssembler::Address(storageGPR, sizeof(int32_t) * index));
3659                 }
3660             }
3661
3662             cellResult(resultGPR, node);
3663             break;
3664         }
3665
3666         flushRegisters();
3667         GPRResult result(this);
3668
3669         callOperation(operationNewArrayBuffer, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), node->startConstant(), node->numConstants());
3670
3671         cellResult(result.gpr(), node);
3672         break;
3673     }
3674
3675     case NewRegexp: {
3676         flushRegisters();
3677         GPRResult resultPayload(this);
3678         GPRResult2 resultTag(this);
3679
3680         callOperation(operationNewRegexp, resultTag.gpr(), resultPayload.gpr(), m_jit.codeBlock()->regexp(node->regexpIndex()));
3681
3682         // FIXME: make the callOperation above explicitly return a cell result, or jitAssert the tag is a cell tag.
3683         cellResult(resultPayload.gpr(), node);
3684         break;
3685     }
3686
3687     case ConvertThis: {
3688         ASSERT(node->child1().useKind() == UntypedUse);
3689
3690         JSValueOperand thisValue(this, node->child1());
3691         GPRReg thisValueTagGPR = thisValue.tagGPR();
3692         GPRReg thisValuePayloadGPR = thisValue.payloadGPR();
3693
3694         flushRegisters();
3695
3696         GPRResult2 resultTag(this);
3697         GPRResult resultPayload(this);
3698         callOperation(operationConvertThis, resultTag.gpr(), resultPayload.gpr(), thisValueTagGPR, thisValuePayloadGPR);
3699
3700         cellResult(resultPayload.gpr(), node);
3701         break;
3702     }
3703
3704     case CreateThis: {
3705         // Note that there is not so much profit to speculate here. The only things we
3706         // speculate on are (1) that it's a cell, since that eliminates cell checks
3707         // later if the proto is reused, and (2) if we have a FinalObject prediction
3708         // then we speculate because we want to get recompiled if it isn't (since
3709         // otherwise we'd start taking slow path a lot).
3710
3711         SpeculateCellOperand callee(this, node->child1());
3712         GPRTemporary result(this);
3713         GPRTemporary allocator(this);
3714         GPRTemporary structure(this);
3715         GPRTemporary scratch(this);
3716
3717         GPRReg calleeGPR = callee.gpr();
3718         GPRReg resultGPR = result.gpr();
3719         GPRReg allocatorGPR = allocator.gpr();
3720         GPRReg structureGPR = structure.gpr();
3721         GPRReg scratchGPR = scratch.gpr();
3722
3723         MacroAssembler::JumpList slowPath;
3724
3725         m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR);
3726         m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR);
3727         slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, allocatorGPR));
3728         emitAllocateJSObject(resultGPR, allocatorGPR, structureGPR, TrustedImmPtr(0), scratchGPR, slowPath);
3729
3730         addSlowPathGenerator(slowPathCall(slowPath, this, operationCreateThis, resultGPR, calleeGPR, node->inlineCapacity()));
3731
3732         cellResult(resultGPR, node);
3733         break;
3734     }
3735
3736     case AllocationProfileWatchpoint: {
3737         jsCast<JSFunction*>(node->function())->addAllocationProfileWatchpoint(speculationWatchpoint());
3738         noResult(node);
3739         break;
3740     }
3741
3742     case NewObject: {
3743         GPRTemporary result(this);
3744         GPRTemporary allocator(this);
3745         GPRTemporary scratch(this);
3746
3747         GPRReg resultGPR = result.gpr();
3748         GPRReg allocatorGPR = allocator.gpr();
3749         GPRReg scratchGPR = scratch.gpr();
3750
3751         MacroAssembler::JumpList slowPath;
3752
3753         Structure* structure = node->structure();
3754         size_t allocationSize = JSObject::allocationSize(structure->inlineCapacity());
3755         MarkedAllocator* allocatorPtr = &m_jit.vm()->heap.allocatorForObjectWithoutDestructor(allocationSize);
3756
3757         m_jit.move(TrustedImmPtr(allocatorPtr), allocatorGPR);
3758         emitAllocateJSObject(resultGPR, allocatorGPR, TrustedImmPtr(structure), TrustedImmPtr(0), scratchGPR, slowPath);
3759
3760         addSlowPathGenerator(slowPathCall(slowPath, this, operationNewObject, resultGPR, structure));
3761
3762         cellResult(resultGPR, node);
3763         break;
3764     }
3765
3766     case GetCallee: {
3767         GPRTemporary result(this);
3768         m_jit.loadPtr(JITCompiler::payloadFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::Callee))), result.gpr());
3769         cellResult(result.gpr(), node);
3770         break;
3771     }
3772
3773     case SetCallee: {
3774         SpeculateCellOperand callee(this, node->child1());
3775         m_jit.storePtr(callee.gpr(), JITCompiler::payloadFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::Callee))));
3776         m_jit.store32(MacroAssembler::TrustedImm32(JSValue::CellTag), JITCompiler::tagFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::Callee))));
3777         noResult(node);
3778         break;
3779     }
3780
3781     case GetScope: {
3782         SpeculateCellOperand function(this, node->child1());
3783         GPRTemporary result(this, function);
3784         m_jit.loadPtr(JITCompiler::Address(function.gpr(), JSFunction::offsetOfScopeChain()), result.gpr());
3785         cellResult(result.gpr(), node);
3786         break;
3787     }
3788
3789     case GetMyScope: {
3790         GPRTemporary result(this);
3791         GPRReg resultGPR = result.gpr();
3792
3793         m_jit.loadPtr(JITCompiler::payloadFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::ScopeChain))), resultGPR);
3794         cellResult(resultGPR, node);
3795         break;
3796     }
3797
3798     case SetMyScope: {
3799         SpeculateCellOperand callee(this, node->child1());
3800         m_jit.storePtr(callee.gpr(), JITCompiler::payloadFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::ScopeChain))));
3801         noResult(node);
3802         break;
3803     }
3804
3805     case SkipTopScope: {
3806         SpeculateCellOperand scope(this, node->child1());
3807         GPRTemporary result(this, scope);
3808         GPRReg resultGPR = result.gpr();
3809         m_jit.move(scope.gpr(), resultGPR);
3810         JITCompiler::Jump activationNotCreated =
3811             m_jit.branchTestPtr(
3812                 JITCompiler::Zero,
3813                 JITCompiler::payloadFor(
3814                     static_cast<VirtualRegister>(m_jit.codeBlock()->activationRegister())));
3815         m_jit.loadPtr(JITCompiler::Address(resultGPR, JSScope::offsetOfNext()), resultGPR);
3816         activationNotCreated.link(&m_jit);
3817         cellResult(resultGPR, node);
3818         break;
3819     }
3820
3821     case SkipScope: {
3822         SpeculateCellOperand scope(this, node->child1());
3823         GPRTemporary result(this, scope);
3824         m_jit.loadPtr(JITCompiler::Address(scope.gpr(), JSScope::offsetOfNext()), result.gpr());
3825         cellResult(result.gpr(), node);
3826         break;
3827     }
3828
3829     case GetScopeRegisters: {
3830         SpeculateCellOperand scope(this, node->child1());
3831         GPRTemporary result(this);
3832         GPRReg scopeGPR = scope.gpr();
3833         GPRReg resultGPR = result.gpr();
3834
3835         m_jit.loadPtr(JITCompiler::Address(scopeGPR, JSVariableObject::offsetOfRegisters()), resultGPR);
3836         storageResult(resultGPR, node);
3837         break;
3838     }
3839     case GetScopedVar: {
3840         StorageOperand registers(this, node->child1());
3841         GPRTemporary resultTag(this);
3842         GPRTemporary resultPayload(this);
3843         GPRReg registersGPR = registers.gpr();
3844         GPRReg resultTagGPR = resultTag.gpr();
3845         GPRReg resultPayloadGPR = resultPayload.gpr();
3846         m_jit.load32(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTagGPR);
3847         m_jit.load32(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR);
3848         jsValueResult(resultTagGPR, resultPayloadGPR, node);
3849         break;
3850     }
3851     case PutScopedVar: {
3852         SpeculateCellOperand scope(this, node->child1());
3853         StorageOperand registers(this, node->child2());
3854         JSValueOperand value(this, node->child3());
3855         GPRTemporary scratchRegister(this);
3856         GPRReg scopeGPR = scope.gpr();
3857         GPRReg registersGPR = registers.gpr();
3858         GPRReg valueTagGPR = value.tagGPR();
3859         GPRReg valuePayloadGPR = value.payloadGPR();
3860         GPRReg scratchGPR = scratchRegister.gpr();
3861
3862         m_jit.store32(valueTagGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
3863         m_jit.store32(valuePayloadGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
3864         writeBarrier(scopeGPR, valueTagGPR, node->child2(), WriteBarrierForVariableAccess, scratchGPR);
3865         noResult(node);
3866         break;
3867     }
3868
3869     case GetById: {
3870         if (!node->prediction()) {
3871             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
3872             break;
3873         }
3874
3875         if (isCellSpeculation(node->child1()->prediction())) {
3876             SpeculateCellOperand base(this, node->child1());
3877             GPRTemporary resultTag(this, base);
3878             GPRTemporary resultPayload(this);
3879
3880             GPRReg baseGPR = base.gpr();
3881             GPRReg resultTagGPR = resultTag.gpr();
3882             GPRReg resultPayloadGPR = resultPayload.gpr();
3883
3884             base.use();
3885
3886             cachedGetById(node->codeOrigin, InvalidGPRReg, baseGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber());
3887
3888             jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3889             break;
3890         }
3891
3892         JSValueOperand base(this, node->child1());
3893         GPRTemporary resultTag(this, base);
3894         GPRTemporary resultPayload(this);
3895
3896         GPRReg baseTagGPR = base.tagGPR();
3897         GPRReg basePayloadGPR = base.payloadGPR();
3898         GPRReg resultTagGPR = resultTag.gpr();
3899         GPRReg resultPayloadGPR = resultPayload.gpr();
3900
3901         base.use();
3902
3903         JITCompiler::Jump notCell = m_jit.branch32(JITCompiler::NotEqual, baseTagGPR, TrustedImm32(JSValue::CellTag));
3904
3905         cachedGetById(node->codeOrigin, baseTagGPR, basePayloadGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), notCell);
3906
3907         jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3908         break;
3909     }
3910
3911     case GetByIdFlush: {
3912         if (!node->prediction()) {
3913             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
3914             break;
3915         }
3916
3917         switch (node->child1().useKind()) {
3918         case CellUse: {
3919             SpeculateCellOperand base(this, node->child1());
3920
3921             GPRReg baseGPR = base.gpr();
3922
3923             GPRResult resultTag(this);
3924             GPRResult2 resultPayload(this);
3925             GPRReg resultTagGPR = resultTag.gpr();
3926             GPRReg resultPayloadGPR = resultPayload.gpr();
3927
3928             base.use();
3929
3930             flushRegisters();
3931
3932             cachedGetById(node->codeOrigin, InvalidGPRReg, baseGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill);
3933
3934             jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3935             break;
3936         }
3937
3938         case UntypedUse: {
3939             JSValueOperand base(this, node->child1());
3940             GPRReg baseTagGPR = base.tagGPR();
3941             GPRReg basePayloadGPR = base.payloadGPR();
3942
3943             GPRResult resultTag(this);
3944             GPRResult2 resultPayload(this);
3945             GPRReg resultTagGPR = resultTag.gpr();
3946             GPRReg resultPayloadGPR = resultPayload.gpr();
3947
3948             base.use();
3949
3950             flushRegisters();
3951
3952             JITCompiler::Jump notCell = m_jit.branch32(JITCompiler::NotEqual, baseTagGPR, TrustedImm32(JSValue::CellTag));
3953
3954             cachedGetById(node->codeOrigin, baseTagGPR, basePayloadGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), notCell, DontSpill);
3955
3956             jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3957             break;
3958         }
3959
3960         default:
3961             RELEASE_ASSERT_NOT_REACHED();
3962             break;
3963         }
3964         break;
3965     }
3966
3967     case GetArrayLength:
3968         compileGetArrayLength(node);
3969         break;
3970
3971     case CheckFunction: {
3972         SpeculateCellOperand function(this, node->child1());
3973         speculationCheck(BadFunction, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, function.gpr(), node->function()));
3974         noResult(node);
3975         break;
3976     }
3977
3978     case CheckExecutable: {
3979         SpeculateCellOperand function(this, node->child1());
3980         speculationCheck(BadExecutable, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, JITCompiler::Address(function.gpr(), JSFunction::offsetOfExecutable()), node->executable()));
3981         noResult(node);
3982         break;
3983     }
3984
3985     case CheckStructure:
3986     case ForwardCheckStructure: {
3987         SpeculateCellOperand base(this, node->child1());
3988
3989         ASSERT(node->structureSet().size());
3990
3991         if (node->structureSet().size() == 1) {
3992             speculationCheck(
3993                 BadCache, JSValueSource::unboxedCell(base.gpr()), 0,
3994                 m_jit.branchWeakPtr(
3995                     JITCompiler::NotEqual,
3996                     JITCompiler::Address(base.gpr(), JSCell::structureOffset()),
3997                     node->structureSet()[0]));
3998         } else {
3999             GPRTemporary structure(this);
4000
4001             m_jit.loadPtr(JITCompiler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr());
4002
4003             JITCompiler::JumpList done;
4004
4005             for (size_t i = 0; i < node->structureSet().size() - 1; ++i)
4006                 done.append(m_jit.branchWeakPtr(JITCompiler::Equal, structure.gpr(), node->structureSet()[i]));
4007
4008             speculationCheck(
4009                 BadCache, JSValueSource::unboxedCell(base.gpr()), 0,
4010                 m_jit.branchWeakPtr(
4011                     JITCompiler::NotEqual, structure.gpr(), node->structureSet().last()));
4012
4013             done.link(&m_jit);
4014         }
4015
4016         noResult(node);
4017         break;
4018     }
4019
4020     case StructureTransitionWatchpoint:
4021     case ForwardStructureTransitionWatchpoint: {
4022         // There is a fascinating question here of what to do about array profiling.
4023         // We *could* try to tell the OSR exit about where the base of the access is.
4024         // The DFG will have kept it alive, though it may not be in a register, and
4025         // we shouldn't really load it since that could be a waste. For now though,
4026         // we'll just rely on the fact that when a watchpoint fires then that's
4027         // quite a hint already.
4028
4029         m_jit.addWeakReference(node->structure());
4030         node->structure()->addTransitionWatchpoint(
4031             speculationWatchpoint(
4032                 node->child1()->op() == WeakJSConstant ? BadWeakConstantCache : BadCache));
4033
4034 #if !ASSERT_DISABLED
4035         SpeculateCellOperand op1(this, node->child1());
4036         JITCompiler::Jump isOK = m_jit.branchPtr(JITCompiler::Equal, JITCompiler::Address(op1.gpr(), JSCell::structureOffset()), TrustedImmPtr(node->structure()));
4037         m_jit.breakpoint();
4038         isOK.link(&m_jit);
4039 #else
4040         speculateCell(node->child1());
4041 #endif
4042
4043         noResult(node);
4044         break;
4045     }
4046
4047     case PhantomPutStructure: {
4048         ASSERT(isKnownCell(node->child1().node()));
4049         ASSERT(node->structureTransitionData().previousStructure->transitionWatchpointSetHasBeenInvalidated());
4050         m_jit.addWeakReferenceTransition(
4051             node->codeOrigin.codeOriginOwner(),
4052             node->structureTransitionData().previousStructure,
4053             node->structureTransitionData().newStructure);
4054         noResult(node);
4055         break;
4056     }
4057
4058     case PutStructure: {
4059         ASSERT(node->structureTransitionData().previousStructure->transitionWatchpointSetHasBeenInvalidated());
4060
4061         SpeculateCellOperand base(this, node->child1());
4062         GPRReg baseGPR = base.gpr();
4063
4064         m_jit.addWeakReferenceTransition(
4065             node->codeOrigin.codeOriginOwner(),
4066             node->structureTransitionData().previousStructure,
4067             node->structureTransitionData().newStructure);
4068
4069 #if ENABLE(WRITE_BARRIER_PROFILING)
4070         // Must always emit this write barrier as the structure transition itself requires it
4071         writeBarrier(baseGPR, node->structureTransitionData().newStructure, WriteBarrierForGenericAccess);
4072 #endif
4073
4074         m_jit.storePtr(MacroAssembler::TrustedImmPtr(node->structureTransitionData().newStructure), MacroAssembler::Address(baseGPR, JSCell::structureOffset()));
4075
4076         noResult(node);
4077         break;
4078     }
4079
4080     case AllocatePropertyStorage:
4081         compileAllocatePropertyStorage(node);
4082         break;
4083
4084     case ReallocatePropertyStorage:
4085         compileReallocatePropertyStorage(node);
4086         break;
4087
4088     case GetButterfly: {
4089         SpeculateCellOperand base(this, node->child1());
4090         GPRTemporary result(this, base);
4091
4092         GPRReg baseGPR = base.gpr();
4093         GPRReg resultGPR = result.gpr();
4094
4095         m_jit.loadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), resultGPR);
4096
4097         storageResult(resultGPR, node);
4098         break;
4099     }
4100
4101     case GetIndexedPropertyStorage: {
4102         compileGetIndexedPropertyStorage(node);
4103         break;
4104     }
4105
4106     case GetByOffset: {
4107         StorageOperand storage(this, node->child1());
4108         GPRTemporary resultTag(this, storage);
4109         GPRTemporary resultPayload(this);
4110
4111         GPRReg storageGPR = storage.gpr();
4112         GPRReg resultTagGPR = resultTag.gpr();
4113         GPRReg resultPayloadGPR = resultPayload.gpr();
4114
4115         StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()];
4116
4117         m_jit.load32(JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR);
4118         m_jit.load32(JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTagGPR);
4119
4120         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4121         break;
4122     }
4123
4124     case PutByOffset: {
4125 #if ENABLE(WRITE_BARRIER_PROFILING)
4126         SpeculateCellOperand base(this, node->child2());
4127 #endif
4128         StorageOperand storage(this, node->child1());
4129         JSValueOperand value(this, node->child3());
4130
4131         GPRReg storageGPR = storage.gpr();
4132         GPRReg valueTagGPR = value.tagGPR();
4133         GPRReg valuePayloadGPR = value.payloadGPR();
4134
4135 #if ENABLE(WRITE_BARRIER_PROFILING)
4136         writeBarrier(base.gpr(), valueTagGPR, node->child3(), WriteBarrierForPropertyAccess);
4137 #endif
4138
4139         StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()];
4140
4141         m_jit.storePtr(valueTagGPR, JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
4142         m_jit.storePtr(valuePayloadGPR, JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
4143
4144         noResult(node);
4145         break;
4146     }
4147
4148     case PutById: {
4149         SpeculateCellOperand base(this, node->child1());
4150         JSValueOperand value(this, node->child2());
4151         GPRTemporary scratch(this);
4152
4153         GPRReg baseGPR = base.gpr();
4154         GPRReg valueTagGPR = value.tagGPR();
4155         GPRReg valuePayloadGPR = value.payloadGPR();
4156         GPRReg scratchGPR = scratch.gpr();
4157
4158         base.use();
4159         value.use();
4160
4161         cachedPutById(node->codeOrigin, baseGPR, valueTagGPR, valuePayloadGPR, node->child2(), scratchGPR, node->identifierNumber(), NotDirect);
4162
4163         noResult(node, UseChildrenCalledExplicitly);
4164         break;
4165     }
4166
4167     case PutByIdDirect: {
4168         SpeculateCellOperand base(this, node->child1());
4169         JSValueOperand value(this, node->child2());
4170         GPRTemporary scratch(this);
4171
4172         GPRReg baseGPR = base.gpr();
4173         GPRReg valueTagGPR = value.tagGPR();
4174         GPRReg valuePayloadGPR = value.payloadGPR();
4175         GPRReg scratchGPR = scratch.gpr();
4176
4177         base.use();
4178         value.use();
4179
4180         cachedPutById(node->codeOrigin, baseGPR, valueTagGPR, valuePayloadGPR, node->child2(), scratchGPR, node->identifierNumber(), Direct);
4181
4182         noResult(node, UseChildrenCalledExplicitly);
4183         break;
4184     }
4185
4186     case GetGlobalVar: {
4187         GPRTemporary resultPayload(this);
4188         GPRTemporary resultTag(this);
4189
4190         m_jit.move(TrustedImmPtr(node->registerPointer()), resultPayload.gpr());
4191         m_jit.load32(JITCompiler::Address(resultPayload.gpr(), OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTag.gpr());
4192         m_jit.load32(JITCompiler::Address(resultPayload.gpr(), OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayload.gpr());
4193
4194         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
4195         break;
4196     }
4197
4198     case PutGlobalVar: {
4199         JSValueOperand value(this, node->child1());
4200         if (Heap::isWriteBarrierEnabled()) {
4201             GPRTemporary scratch(this);
4202             GPRReg scratchReg = scratch.gpr();
4203
4204             writeBarrier(m_jit.globalObjectFor(node->codeOrigin), value.tagGPR(), node->child1(), WriteBarrierForVariableAccess, scratchReg);
4205         }
4206
4207         // FIXME: if we happen to have a spare register - and _ONLY_ if we happen to have
4208         // a spare register - a good optimization would be to put the register pointer into
4209         // a register and then do a zero offset store followed by a four-offset store (or
4210         // vice-versa depending on endianness).
4211         m_jit.store32(value.tagGPR(), node->registerPointer()->tagPointer());
4212         m_jit.store32(value.payloadGPR(), node->registerPointer()->payloadPointer());
4213
4214         noResult(node);
4215         break;
4216     }
4217
4218     case PutGlobalVarCheck: {
4219         JSValueOperand value(this, node->child1());
4220
4221         WatchpointSet* watchpointSet =
4222             m_jit.globalObjectFor(node->codeOrigin)->symbolTable()->get(
4223                 identifier(node->identifierNumberForCheck())->impl()).watchpointSet();
4224         addSlowPathGenerator(
4225             slowPathCall(
4226                 m_jit.branchTest8(
4227                     JITCompiler::NonZero,
4228                     JITCompiler::AbsoluteAddress(watchpointSet->addressOfIsWatched())),
4229                 this, operationNotifyGlobalVarWrite, NoResult, watchpointSet));
4230
4231         if (Heap::isWriteBarrierEnabled()) {
4232             GPRTemporary scratch(this);
4233             GPRReg scratchReg = scratch.gpr();
4234
4235             writeBarrier(m_jit.globalObjectFor(node->codeOrigin), value.tagGPR(), node->child1(), WriteBarrierForVariableAccess, scratchReg);
4236         }
4237
4238         // FIXME: if we happen to have a spare register - and _ONLY_ if we happen to have
4239         // a spare register - a good optimization would be to put the register pointer into
4240         // a register and then do a zero offset store followed by a four-offset store (or
4241         // vice-versa depending on endianness).
4242         m_jit.store32(value.tagGPR(), node->registerPointer()->tagPointer());
4243         m_jit.store32(value.payloadGPR(), node->registerPointer()->payloadPointer());
4244
4245         noResult(node);
4246         break;
4247     }
4248
4249     case GlobalVarWatchpoint: {
4250         m_jit.globalObjectFor(node->codeOrigin)->symbolTable()->get(
4251             identifier(node->identifierNumberForCheck())->impl()).addWatchpoint(
4252                 speculationWatchpoint());
4253
4254 #if DFG_ENABLE(JIT_ASSERT)
4255         GPRTemporary scratch(this);
4256         GPRReg scratchGPR = scratch.gpr();
4257         m_jit.load32(node->registerPointer()->tagPointer(), scratchGPR);
4258         JITCompiler::Jump notOK = m_jit.branch32(
4259             JITCompiler::NotEqual, scratchGPR,
4260             TrustedImm32(node->registerPointer()->get().tag()));
4261         m_jit.load32(node->registerPointer()->payloadPointer(), scratchGPR);
4262         JITCompiler::Jump ok = m_jit.branch32(
4263             JITCompiler::Equal, scratchGPR,
4264             TrustedImm32(node->registerPointer()->get().payload()));
4265         notOK.link(&m_jit);
4266         m_jit.breakpoint();
4267         ok.link(&m_jit);
4268 #endif
4269
4270         noResult(node);
4271         break;
4272     }
4273
4274     case CheckHasInstance: {
4275         SpeculateCellOperand base(this, node->child1());
4276         GPRTemporary structure(this);
4277
4278         // Speculate that base 'ImplementsDefaultHasInstance'.
4279         m_jit.loadPtr(MacroAssembler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr());
4280         speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(structure.gpr(), Structure::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance)));
4281
4282         noResult(node);
4283         break;
4284     }
4285
4286     case InstanceOf: {
4287         compileInstanceOf(node);
4288         break;
4289     }
4290
4291     case IsUndefined: {
4292         JSValueOperand value(this, node->child1());
4293         GPRTemporary result(this);
4294         GPRTemporary localGlobalObject(this);
4295         GPRTemporary remoteGlobalObject(this);
4296
4297         JITCompiler::Jump isCell = m_jit.branch32(JITCompiler::Equal, value.tagGPR(), JITCompiler::TrustedImm32(JSValue::CellTag));
4298
4299         m_jit.compare32(JITCompiler::Equal, value.tagGPR(), TrustedImm32(JSValue::UndefinedTag), result.gpr());
4300         JITCompiler::Jump done = m_jit.jump();
4301
4302         isCell.link(&m_jit);
4303         JITCompiler::Jump notMasqueradesAsUndefined;
4304         if (m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
4305             m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
4306             m_jit.move(TrustedImm32(0), result.gpr());
4307             notMasqueradesAsUndefined = m_jit.jump();
4308         } else {
4309             m_jit.loadPtr(JITCompiler::Address(value.payloadGPR(), JSCell::structureOffset()), result.gpr());
4310             JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(result.gpr(), Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined));
4311             m_jit.move(TrustedImm32(0), result.gpr());
4312             notMasqueradesAsUndefined = m_jit.jump();
4313
4314             isMasqueradesAsUndefined.link(&m_jit);
4315             GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
4316             GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
4317             m_jit.move(TrustedImmPtr(m_jit.globalObjectFor(node->codeOrigin)), localGlobalObjectGPR);
4318             m_jit.loadPtr(JITCompiler::Address(result.gpr(), Structure::globalObjectOffset()), remoteGlobalObjectGPR);
4319             m_jit.compare32(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, result.gpr());
4320         }
4321
4322         notMasqueradesAsUndefined.link(&m_jit);
4323         done.link(&m_jit);
4324         booleanResult(result.gpr(), node);
4325         break;
4326     }
4327
4328     case IsBoolean: {
4329         JSValueOperand value(this, node->child1());
4330         GPRTemporary result(this, value);
4331
4332         m_jit.compare32(JITCompiler::Equal, value.tagGPR(), JITCompiler::TrustedImm32(JSValue::BooleanTag), result.gpr());
4333         booleanResult(result.gpr(), node);
4334         break;
4335     }
4336
4337     case IsNumber: {
4338         JSValueOperand value(this, node->child1());
4339         GPRTemporary result(this, value);
4340
4341         m_jit.add32(TrustedImm32(1), value.tagGPR(), result.gpr());
4342         m_jit.compare32(JITCompiler::Below, result.gpr(), JITCompiler::TrustedImm32(JSValue::LowestTag + 1), result.gpr());
4343         booleanResult(result.gpr(), node);
4344         break;
4345     }
4346
4347     case IsString: {
4348         JSValueOperand value(this, node->child1());
4349         GPRTemporary result(this, value);
4350
4351         JITCompiler::Jump isNotCell = m_jit.branch32(JITCompiler::NotEqual, value.tagGPR(), JITCompiler::TrustedImm32(JSValue::CellTag));
4352
4353         m_jit.loadPtr(JITCompiler::Address(value.payloadGPR(), JSCell::structureOffset()), result.gpr());
4354         m_jit.compare8(JITCompiler::Equal, JITCompiler::Address(result.gpr(), Structure::typeInfoTypeOffset()), TrustedImm32(StringType), result.gpr());
4355         JITCompiler::Jump done = m_jit.jump();
4356
4357         isNotCell.link(&m_jit);
4358         m_jit.move(TrustedImm32(0), result.gpr());
4359
4360         done.link(&m_jit);
4361         booleanResult(result.gpr(), node);
4362         break;
4363     }
4364
4365     case IsObject: {
4366         JSValueOperand value(this, node->child1());
4367         GPRReg valueTagGPR = value.tagGPR();
4368         GPRReg valuePayloadGPR = value.payloadGPR();
4369         GPRResult result(this);
4370         GPRReg resultGPR = result.gpr();
4371         flushRegisters();
4372         callOperation(operationIsObject, resultGPR, valueTagGPR, valuePayloadGPR);
4373         booleanResult(result.gpr(), node);
4374         break;
4375     }
4376
4377     case IsFunction: {
4378         JSValueOperand value(this, node->child1());
4379         GPRReg valueTagGPR = value.tagGPR();
4380         GPRReg valuePayloadGPR = value.payloadGPR();
4381         GPRResult result(this);
4382         GPRReg resultGPR = result.gpr();
4383         flushRegisters();
4384         callOperation(operationIsFunction, resultGPR, valueTagGPR, valuePayloadGPR);
4385         booleanResult(result.gpr(), node);
4386         break;
4387     }
4388     case TypeOf: {
4389         JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
4390         GPRReg tagGPR = value.tagGPR();
4391         GPRReg payloadGPR = value.payloadGPR();
4392         GPRTemporary temp(this);
4393         GPRReg tempGPR = temp.gpr();
4394         GPRResult result(this);
4395         GPRReg resultGPR = result.gpr();
4396         JITCompiler::JumpList doneJumps;
4397
4398         flushRegisters();
4399
4400         ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == CellUse || node->child1().useKind() == StringUse);
4401
4402         JITCompiler::Jump isNotCell = m_jit.branch32(JITCompiler::NotEqual, tagGPR, JITCompiler::TrustedImm32(JSValue::CellTag));
4403         if (node->child1().useKind() != UntypedUse)
4404             speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), node->child1(), isNotCell);
4405
4406         if (!node->child1()->shouldSpeculateObject() || node->child1().useKind() == StringUse) {
4407             m_jit.loadPtr(JITCompiler::Address(payloadGPR, JSCell::structureOffset()), tempGPR);
4408             JITCompiler::Jump notString = m_jit.branch8(JITCompiler::NotEqual, JITCompiler::Address(tempGPR, Structure::typeInfoTypeOffset()), TrustedImm32(StringType));
4409             if (node->child1().useKind() == StringUse)
4410                 DFG_TYPE_CHECK(JSValueRegs(tagGPR, payloadGPR), node->child1(), SpecString, notString);
4411             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.stringString()), resultGPR);
4412             doneJumps.append(m_jit.jump());
4413             if (node->child1().useKind() != StringUse) {
4414                 notString.link(&m_jit);
4415                 callOperation(operationTypeOf, resultGPR, payloadGPR);
4416                 doneJumps.append(m_jit.jump());
4417             }
4418         } else {
4419             callOperation(operationTypeOf, resultGPR, payloadGPR);
4420             doneJumps.append(m_jit.jump());
4421         }
4422
4423         if (node->child1().useKind() == UntypedUse) {
4424             isNotCell.link(&m_jit);
4425
4426             m_jit.add32(TrustedImm32(1), tagGPR, tempGPR);
4427             JITCompiler::Jump notNumber = m_jit.branch32(JITCompiler::AboveOrEqual, tempGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
4428             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.numberString()), resultGPR);
4429             doneJumps.append(m_jit.jump());
4430             notNumber.link(&m_jit);
4431
4432             JITCompiler::Jump notUndefined = m_jit.branch32(JITCompiler::NotEqual, tagGPR, TrustedImm32(JSValue::UndefinedTag));
4433             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.undefinedString()), resultGPR);
4434             doneJumps.append(m_jit.jump());
4435             notUndefined.link(&m_jit);
4436
4437             JITCompiler::Jump notNull = m_jit.branch32(JITCompiler::NotEqual, tagGPR, TrustedImm32(JSValue::NullTag));
4438             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.objectString()), resultGPR);
4439             doneJumps.append(m_jit.jump());
4440             notNull.link(&m_jit);
4441
4442             // Only boolean left
4443             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.booleanString()), resultGPR);
4444         }
4445         doneJumps.link(&m_jit);
4446         cellResult(resultGPR, node);
4447         break;
4448     }
4449
4450     case Phi:
4451     case Flush:
4452         break;
4453
4454     case Breakpoint:
4455 #if ENABLE(DEBUG_WITH_BREAKPOINT)
4456         m_jit.breakpoint();
4457 #else
4458         RELEASE_ASSERT_NOT_REACHED();
4459 #endif
4460         break;
4461
4462     case Call:
4463     case Construct:
4464         emitCall(node);
4465         break;
4466
4467     case Resolve: {
4468         flushRegisters();
4469         GPRResult resultPayload(this);
4470         GPRResult2 resultTag(this);
4471         ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()];
4472         callOperation(operationResolve, resultTag.gpr(), resultPayload.gpr(), identifier(data.identifierNumber), data.resolveOperations);
4473         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
4474         break;
4475     }
4476
4477     case ResolveBase: {
4478         flushRegisters();
4479         GPRResult resultPayload(this);
4480         GPRResult2 resultTag(this);
4481         ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()];
4482         callOperation(operationResolveBase, resultTag.gpr(), resultPayload.gpr(), identifier(data.identifierNumber), data.resolveOperations, data.putToBaseOperation);
4483         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
4484         break;
4485     }
4486
4487     case ResolveBaseStrictPut: {
4488         flushRegisters();
4489         GPRResult resultPayload(this);
4490         GPRResult2 resultTag(this);
4491         ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()];
4492         callOperation(operationResolveBaseStrictPut, resultTag.gpr(), resultPayload.gpr(), identifier(data.identifierNumber), data.resolveOperations, data.putToBaseOperation);
4493         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
4494         break;
4495     }
4496
4497     case ResolveGlobal: {
4498         GPRTemporary globalObject(this);
4499         GPRTemporary resolveInfo(this);
4500         GPRTemporary resultTag(this);
4501         GPRTemporary resultPayload(this);
4502
4503         GPRReg globalObjectGPR = globalObject.gpr();
4504         GPRReg resolveInfoGPR = resolveInfo.gpr();
4505         GPRReg resultTagGPR = resultTag.gpr();
4506         GPRReg resultPayloadGPR = resultPayload.gpr();
4507
4508         ResolveGlobalData& data = m_jit.graph().m_resolveGlobalData[node->resolveGlobalDataIndex()];
4509         ResolveOperation* resolveOperationAddress = &(data.resolveOperations->data()[data.resolvePropertyIndex]);
4510
4511         // Check Structure of global object
4512         m_jit.move(JITCompiler::TrustedImmPtr(m_jit.globalObjectFor(node->codeOrigin)), globalObjectGPR);
4513         m_jit.move(JITCompiler::TrustedImmPtr(resolveOperationAddress), resolveInfoGPR);
4514         m_jit.loadPtr(JITCompiler::Address(resolveInfoGPR, OBJECT_OFFSETOF(ResolveOperation, m_structure)), resultPayloadGPR);
4515
4516         JITCompiler::Jump structuresNotMatch = m_jit.branchPtr(JITCompiler::NotEqual, resultPayloadGPR, JITCompiler::Address(globalObjectGPR, JSCell::structureOffset()));
4517
4518         // Fast case
4519         m_jit.loadPtr(JITCompiler::Address(globalObjectGPR, JSObject::butterflyOffset()), resultPayloadGPR);
4520         m_jit.load32(JITCompiler::Address(resolveInfoGPR, OBJECT_OFFSETOF(ResolveOperation, m_offset)), resolveInfoGPR);
4521 #if DFG_ENABLE(JIT_ASSERT)
4522         JITCompiler::Jump isOutOfLine = m_jit.branch32(JITCompiler::GreaterThanOrEqual, resolveInfoGPR, TrustedImm32(firstOutOfLineOffset));
4523         m_jit.breakpoint();
4524         isOutOfLine.link(&m_jit);
4525 #endif
4526         m_jit.neg32(resolveInfoGPR);
4527         m_jit.signExtend32ToPtr(resolveInfoGPR, resolveInfoGPR);
4528         m_jit.load32(JITCompiler::BaseIndex(resultPayloadGPR, resolveInfoGPR, JITCompiler::TimesEight, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag) + (firstOutOfLineOffset - 2) * static_cast<ptrdiff_t>(sizeof(JSValue))), resultTagGPR);
4529         m_jit.load32(JITCompiler::BaseIndex(resultPayloadGPR, resolveInfoGPR, JITCompiler::TimesEight, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload) + (firstOutOfLineOffset - 2) * static_cast<ptrdiff_t>(sizeof(JSValue))), resultPayloadGPR);
4530
4531         addSlowPathGenerator(
4532             slowPathCall(
4533                 structuresNotMatch, this, operationResolveGlobal,
4534                 JSValueRegs(resultTagGPR, resultPayloadGPR), resolveInfoGPR, globalObjectGPR,
4535                 &m_jit.codeBlock()->identifier(data.identifierNumber)));
4536
4537         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4538         break;
4539     }
4540
4541     case CreateActivation: {
4542         JSValueOperand value(this, node->child1());
4543         GPRTemporary result(this, value, false);
4544
4545         GPRReg valueTagGPR = value.tagGPR();
4546         GPRReg valuePayloadGPR = value.payloadGPR();
4547         GPRReg resultGPR = result.gpr();
4548
4549         m_jit.move(valuePayloadGPR, resultGPR);
4550
4551         JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
4552
4553         addSlowPathGenerator(
4554             slowPathCall(notCreated, this, operationCreateActivation, resultGPR));
4555
4556         cellResult(resultGPR, node);
4557         break;
4558     }
4559
4560     case CreateArguments: {
4561         JSValueOperand value(this, node->child1());
4562         GPRTemporary result(this, value, false);
4563
4564         GPRReg valueTagGPR = value.tagGPR();
4565         GPRReg valuePayloadGPR = value.payloadGPR();
4566         GPRReg resultGPR = result.gpr();
4567
4568         m_jit.move(valuePayloadGPR, resultGPR);
4569
4570         JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
4571
4572         if (node->codeOrigin.inlineCallFrame) {
4573             addSlowPathGenerator(
4574                 slowPathCall(
4575                     notCreated, this, operationCreateInlinedArguments, resultGPR,
4576                     node->codeOrigin.inlineCallFrame));
4577         } else {
4578             addSlowPathGenerator(
4579                 slowPathCall(notCreated, this, operationCreateArguments, resultGPR));
4580         }
4581
4582         cellResult(resultGPR, node);
4583         break;
4584     }
4585
4586     case TearOffActivation: {
4587         JSValueOperand activationValue(this, node->child1());
4588         GPRTemporary scratch(this);
4589
4590         GPRReg activationValueTagGPR = activationValue.tagGPR();
4591         GPRReg activationValuePayloadGPR = activationValue.payloadGPR();
4592         GPRReg scratchGPR = scratch.gpr();
4593
4594         JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, activationValueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
4595
4596         SharedSymbolTable* symbolTable = m_jit.symbolTableFor(node->codeOrigin);
4597         int registersOffset = JSActivation::registersOffset(symbolTable);
4598
4599         int captureEnd = symbolTable->captureEnd();
4600         for (int i = symbolTable->captureStart(); i < captureEnd; ++i) {
4601             m_jit.loadPtr(
4602                 JITCompiler::Address(
4603                     GPRInfo::callFrameRegister, i * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4604                 scratchGPR);
4605             m_jit.storePtr(
4606                 scratchGPR, JITCompiler::Address(
4607                     activationValuePayloadGPR, registersOffset + i * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
4608             m_jit.loadPtr(
4609                 JITCompiler::Address(
4610                     GPRInfo::callFrameRegister, i * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4611                 scratchGPR);
4612             m_jit.storePtr(
4613                 scratchGPR, JITCompiler::Address(
4614                     activationValuePayloadGPR, registersOffset + i * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
4615         }
4616         m_jit.addPtr(TrustedImm32(registersOffset), activationValuePayloadGPR, scratchGPR);
4617         m_jit.storePtr(scratchGPR, JITCompiler::Address(activationValuePayloadGPR, JSActivation::offsetOfRegisters()));
4618
4619         notCreated.link(&m_jit);
4620         noResult(node);
4621         break;
4622     }
4623
4624     case TearOffArguments: {
4625         JSValueOperand unmodifiedArgumentsValue(this, node->child1());
4626         JSValueOperand activationValue(this, node->child2());
4627         GPRReg unmodifiedArgumentsValuePayloadGPR = unmodifiedArgumentsValue.payloadGPR();
4628         GPRReg activationValuePayloadGPR = activationValue.payloadGPR();
4629
4630         JITCompiler::Jump created = m_jit.branchTest32(
4631             JITCompiler::NonZero, unmodifiedArgumentsValuePayloadGPR);
4632
4633         if (node->codeOrigin.inlineCallFrame) {
4634             addSlowPathGenerator(
4635                 slowPathCall(
4636                     created, this, operationTearOffInlinedArguments, NoResult,
4637                     unmodifiedArgumentsValuePayloadGPR, activationValuePayloadGPR, node->codeOrigin.inlineCallFrame));
4638         } else {
4639             addSlowPathGenerator(
4640                 slowPathCall(
4641                     created, this, operationTearOffArguments, NoResult,
4642                     unmodifiedArgumentsValuePayloadGPR, activationValuePayloadGPR));
4643         }
4644
4645         noResult(node);
4646         break;
4647     }
4648
4649     case CheckArgumentsNotCreated: {
4650         ASSERT(!isEmptySpeculation(
4651             m_state.variables().operand(
4652                 m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type));
4653         speculationCheck(
4654             Uncountable, JSValueRegs(), 0,
4655             m_jit.branch32(
4656                 JITCompiler::NotEqual,
4657                 JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4658                 TrustedImm32(JSValue::EmptyValueTag)));
4659         noResult(node);
4660         break;
4661     }
4662
4663     case GetMyArgumentsLength: {
4664         GPRTemporary result(this);
4665         GPRReg resultGPR = result.gpr();
4666
4667         if (!isEmptySpeculation(
4668                 m_state.variables().operand(
4669                     m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) {
4670             speculationCheck(
4671                 ArgumentsEscaped, JSValueRegs(), 0,
4672                 m_jit.branch32(
4673                     JITCompiler::NotEqual,
4674                     JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4675                     TrustedImm32(JSValue::EmptyValueTag)));
4676         }
4677
4678         ASSERT(!node->codeOrigin.inlineCallFrame);
4679         m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR);
4680         m_jit.sub32(TrustedImm32(1), resultGPR);
4681         integerResult(resultGPR, node);
4682         break;
4683     }
4684
4685     case GetMyArgumentsLengthSafe: {
4686         GPRTemporary resultPayload(this);
4687         GPRTemporary resultTag(this);
4688         GPRReg resultPayloadGPR = resultPayload.gpr();
4689         GPRReg resultTagGPR = resultTag.gpr();
4690
4691         JITCompiler::Jump created = m_jit.branch32(
4692             JITCompiler::NotEqual,
4693             JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4694             TrustedImm32(JSValue::EmptyValueTag));
4695
4696         if (node->codeOrigin.inlineCallFrame) {
4697             m_jit.move(
4698                 Imm32(node->codeOrigin.inlineCallFrame->arguments.size() - 1),
4699                 resultPayloadGPR);
4700         } else {
4701             m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultPayloadGPR);
4702             m_jit.sub32(TrustedImm32(1), resultPayloadGPR);
4703         }
4704         m_jit.move(TrustedImm32(JSValue::Int32Tag), resultTagGPR);
4705
4706         // FIXME: the slow path generator should perform a forward speculation that the
4707         // result is an integer. For now we postpone the speculation by having this return
4708         // a JSValue.
4709
4710         addSlowPathGenerator(
4711             slowPathCall(
4712                 created, this, operationGetArgumentsLength,
4713                 JSValueRegs(resultTagGPR, resultPayloadGPR),
4714                 m_jit.argumentsRegisterFor(node->codeOrigin)));
4715
4716         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4717         break;
4718     }
4719
4720     case GetMyArgumentByVal: {
4721         SpeculateStrictInt32Operand index(this, node->child1());
4722         GPRTemporary resultPayload(this);
4723         GPRTemporary resultTag(this);
4724         GPRReg indexGPR = index.gpr();
4725         GPRReg resultPayloadGPR = resultPayload.gpr();
4726         GPRReg resultTagGPR = resultTag.gpr();
4727
4728         if (!isEmptySpeculation(
4729                 m_state.variables().operand(
4730                     m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) {
4731             speculationCheck(
4732                 ArgumentsEscaped, JSValueRegs(), 0,
4733                 m_jit.branch32(
4734                     JITCompiler::NotEqual,
4735                     JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4736                     TrustedImm32(JSValue::EmptyValueTag)));
4737         }
4738
4739         m_jit.add32(TrustedImm32(1), indexGPR, resultPayloadGPR);
4740
4741         if (node->codeOrigin.inlineCallFrame) {
4742             speculationCheck(
4743                 Uncountable, JSValueRegs(), 0,
4744                 m_jit.branch32(
4745                     JITCompiler::AboveOrEqual,
4746                     resultPayloadGPR,
4747                     Imm32(node->codeOrigin.inlineCallFrame->arguments.size())));
4748         } else {
4749             speculationCheck(
4750                 Uncountable, JSValueRegs(), 0,
4751                 m_jit.branch32(
4752                     JITCompiler::AboveOrEqual,
4753                     resultPayloadGPR,
4754                     JITCompiler::payloadFor(JSStack::ArgumentCount)));
4755         }
4756
4757         JITCompiler::JumpList slowArgument;
4758         JITCompiler::JumpList slowArgumentOutOfBounds;
4759         if (const SlowArgument* slowArguments = m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) {
4760             slowArgumentOutOfBounds.append(
4761                 m_jit.branch32(
4762                     JITCompiler::AboveOrEqual, indexGPR,
4763                     Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount())));
4764
4765             COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes);
4766             m_jit.move(ImmPtr(slowArguments), resultPayloadGPR);
4767             m_jit.load32(
4768                 JITCompiler::BaseIndex(
4769                     resultPayloadGPR, indexGPR, JITCompiler::TimesEight,
4770                     OBJECT_OFFSETOF(SlowArgument, index)),
4771                 resultPayloadGPR);
4772
4773             m_jit.load32(
4774                 JITCompiler::BaseIndex(
4775                     GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4776                     m_jit.offsetOfLocals(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4777                 resultTagGPR);
4778             m_jit.load32(
4779                 JITCompiler::BaseIndex(
4780                     GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4781                     m_jit.offsetOfLocals(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4782                 resultPayloadGPR);
4783             slowArgument.append(m_jit.jump());
4784         }
4785         slowArgumentOutOfBounds.link(&m_jit);
4786
4787         m_jit.neg32(resultPayloadGPR);
4788
4789         m_jit.load32(
4790             JITCompiler::BaseIndex(
4791                 GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4792                 m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4793             resultTagGPR);
4794         m_jit.load32(
4795             JITCompiler::BaseIndex(
4796                 GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4797                 m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4798             resultPayloadGPR);
4799
4800         slowArgument.link(&m_jit);
4801         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4802         break;
4803     }
4804     case GetMyArgumentByValSafe: {
4805         SpeculateStrictInt32Operand index(this, node->child1());
4806         GPRTemporary resultPayload(this);
4807         GPRTemporary resultTag(this);
4808         GPRReg indexGPR = index.gpr();
4809         GPRReg resultPayloadGPR = resultPayload.gpr();
4810         GPRReg resultTagGPR = resultTag.gpr();
4811
4812         JITCompiler::JumpList slowPath;
4813         slowPath.append(
4814             m_jit.branch32(
4815                 JITCompiler::NotEqual,
4816                 JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4817                 TrustedImm32(JSValue::EmptyValueTag)));
4818
4819         m_jit.add32(TrustedImm32(1), indexGPR, resultPayloadGPR);
4820         if (node->codeOrigin.inlineCallFrame) {
4821             slowPath.append(
4822                 m_jit.branch32(
4823                     JITCompiler::AboveOrEqual,
4824                     resultPayloadGPR,
4825                     Imm32(node->codeOrigin.inlineCallFrame->arguments.size())));
4826         } else {
4827             slowPath.append(
4828                 m_jit.branch32(
4829                     JITCompiler::AboveOrEqual,
4830                     resultPayloadGPR,
4831                     JITCompiler::payloadFor(JSStack::ArgumentCount)));
4832         }
4833
4834         JITCompiler::JumpList slowArgument;
4835         JITCompiler::JumpList slowArgumentOutOfBounds;
4836         if (const SlowArgument* slowArguments = m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) {
4837             slowArgumentOutOfBounds.append(
4838                 m_jit.branch32(
4839                     JITCompiler::AboveOrEqual, indexGPR,
4840                     Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount())));
4841
4842             COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes);
4843             m_jit.move(ImmPtr(slowArguments), resultPayloadGPR);
4844             m_jit.load32(
4845                 JITCompiler::BaseIndex(
4846                     resultPayloadGPR, indexGPR, JITCompiler::TimesEight,
4847                     OBJECT_OFFSETOF(SlowArgument, index)),
4848                 resultPayloadGPR);
4849             m_jit.load32(
4850                 JITCompiler::BaseIndex(
4851                     GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4852                     m_jit.offsetOfLocals(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4853                 resultTagGPR);
4854             m_jit.load32(
4855                 JITCompiler::BaseIndex(
4856                     GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4857                     m_jit.offsetOfLocals(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4858                 resultPayloadGPR);
4859             slowArgument.append(m_jit.jump());
4860         }
4861         slowArgumentOutOfBounds.link(&m_jit);
4862
4863         m_jit.neg32(resultPayloadGPR);
4864
4865         m_jit.load32(
4866             JITCompiler::BaseIndex(
4867                 GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4868                 m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4869             resultTagGPR);
4870         m_jit.load32(
4871             JITCompiler::BaseIndex(
4872                 GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4873                 m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4874             resultPayloadGPR);
4875
4876         if (node->codeOrigin.inlineCallFrame) {
4877             addSlowPathGenerator(
4878                 slowPathCall(
4879                     slowPath, this, operationGetInlinedArgumentByVal,
4880                     JSValueRegs(resultTagGPR, resultPayloadGPR),
4881                     m_jit.argumentsRegisterFor(node->codeOrigin),
4882                     node->codeOrigin.inlineCallFrame, indexGPR));
4883         } else {
4884             addSlowPathGenerator(
4885                 slowPathCall(
4886                     slowPath, this, operationGetArgumentByVal,
4887                     JSValueRegs(resultTagGPR, resultPayloadGPR),
4888                     m_jit.argumentsRegisterFor(node->codeOrigin), indexGPR));
4889         }
4890
4891         slowArgument.link(&m_jit);
4892         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4893         break;
4894     }
4895
4896     case NewFunctionNoCheck:
4897         compileNewFunctionNoCheck(node);
4898         break;
4899
4900     case NewFunction: {
4901         JSValueOperand value(this, node->child1());
4902         GPRTemporary resultTag(this, value);
4903         GPRTemporary resultPayload(this, value, false);
4904
4905         GPRReg valueTagGPR = value.tagGPR();
4906         GPRReg valuePayloadGPR = value.payloadGPR();
4907         GPRReg resultTagGPR = resultTag.gpr();
4908         GPRReg resultPayloadGPR = resultPayload.gpr();
4909
4910         m_jit.move(valuePayloadGPR, resultPayloadGPR);
4911         m_jit.move(valueTagGPR, resultTagGPR);
4912
4913         JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
4914
4915         addSlowPathGenerator(
4916             slowPathCall(
4917                 notCreated, this, operationNewFunction, JSValueRegs(resultTagGPR, resultPayloadGPR),
4918                 m_jit.codeBlock()->functionDecl(node->functionDeclIndex())));
4919
4920         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4921         break;
4922     }
4923
4924     case NewFunctionExpression:
4925         compileNewFunctionExpression(node);
4926         break;
4927
4928     case GarbageValue:
4929         // We should never get to the point of code emission for a GarbageValue
4930         CRASH();
4931         break;
4932
4933     case ForceOSRExit: {
4934         terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
4935         break;
4936     }
4937
4938     case CheckWatchdogTimer:
4939         speculationCheck(
4940             WatchdogTimerFired, JSValueRegs(), 0,
4941             m_jit.branchTest8(
4942                 JITCompiler::NonZero,
4943                 JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog.timerDidFireAddress())));
4944         break;
4945
4946     case CountExecution:
4947         m_jit.add64(TrustedImm32(1), MacroAssembler::AbsoluteAddress(node->executionCounter()->address()));
4948         break;
4949
4950     case Phantom:
4951         DFG_NODE_DO_TO_CHILDREN(m_jit.graph(), node, speculate);
4952         noResult(node);
4953         break;
4954
4955     case PhantomLocal:
4956         // This is a no-op.
4957         noResult(node);
4958         break;
4959
4960     case Nop:
4961     case LastNodeType:
4962         RELEASE_ASSERT_NOT_REACHED();
4963         break;
4964     }
4965
4966 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
4967     m_jit.clearRegisterAllocationOffsets();
4968 #endif
4969
4970     if (!m_compileOkay)
4971         return;
4972
4973     if (node->hasResult() && node->mustGenerate())
4974         use(node);
4975 }
4976
4977 #endif
4978
4979 } } // namespace JSC::DFG
4980
4981 #endif