dfg/DFGSpeculativeJIT32_64.cpp

   1 /*
   2  * Copyright (C) 2011, 2012, 2013 Apple Inc. All rights reserved.
   3  * Copyright (C) 2011 Intel Corporation. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  15  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  21  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  22  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  24  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include "config.h"
  28 #include "DFGSpeculativeJIT.h"
  29
  30 #if ENABLE(DFG_JIT)
  31
  32 #include "ArrayPrototype.h"
  33 #include "DFGCallArrayAllocatorSlowPathGenerator.h"
  34 #include "DFGSlowPathGenerator.h"
  35 #include "JSActivation.h"
  36 #include "ObjectPrototype.h"
  37 #include "Operations.h"
  38
  39 namespace JSC { namespace DFG {
  40
  41 #if USE(JSVALUE32_64)
  42
  43 GPRReg SpeculativeJIT::fillInteger(Edge edge, DataFormat& returnFormat)
  44 {
  45     ASSERT(!needsTypeCheck(edge, SpecInt32));
  46
  47     VirtualRegister virtualRegister = edge->virtualRegister();
  48     GenerationInfo& info = m_generationInfo[virtualRegister];
  49
  50     if (info.registerFormat() == DataFormatNone) {
  51         GPRReg gpr = allocate();
  52
  53         if (edge->hasConstant()) {
  54             m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
  55             if (isInt32Constant(edge.node()))
  56                 m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
  57             else if (isNumberConstant(edge.node()))
  58                 RELEASE_ASSERT_NOT_REACHED();
  59             else {
  60                 ASSERT(isJSConstant(edge.node()));
  61                 JSValue jsValue = valueOfJSConstant(edge.node());
  62                 m_jit.move(MacroAssembler::Imm32(jsValue.payload()), gpr);
  63             }
  64         } else {
  65             ASSERT(info.spillFormat() == DataFormatJS || info.spillFormat() == DataFormatJSInteger || info.spillFormat() == DataFormatInteger);
  66             m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
  67             m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr);
  68         }
  69
  70         info.fillInteger(*m_stream, gpr);
  71         returnFormat = DataFormatInteger;
  72         return gpr;
  73     }
  74
  75     switch (info.registerFormat()) {
  76     case DataFormatNone:
  77         // Should have filled, above.
  78     case DataFormatJSDouble:
  79     case DataFormatDouble:
  80     case DataFormatJS:
  81     case DataFormatCell:
  82     case DataFormatJSCell:
  83     case DataFormatBoolean:
  84     case DataFormatJSBoolean:
  85     case DataFormatStorage:
  86         // Should only be calling this function if we know this operand to be integer.
  87         RELEASE_ASSERT_NOT_REACHED();
  88
  89     case DataFormatJSInteger: {
  90         GPRReg tagGPR = info.tagGPR();
  91         GPRReg payloadGPR = info.payloadGPR();
  92         m_gprs.lock(tagGPR);
  93         m_jit.jitAssertIsJSInt32(tagGPR);
  94         m_gprs.unlock(tagGPR);
  95         m_gprs.lock(payloadGPR);
  96         m_gprs.release(tagGPR);
  97         m_gprs.release(payloadGPR);
  98         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderInteger);
  99         info.fillInteger(*m_stream, payloadGPR);
 100         returnFormat = DataFormatInteger;
 101         return payloadGPR;
 102     }
 103
 104     case DataFormatInteger: {
 105         GPRReg gpr = info.gpr();
 106         m_gprs.lock(gpr);
 107         m_jit.jitAssertIsInt32(gpr);
 108         returnFormat = DataFormatInteger;
 109         return gpr;
 110     }
 111
 112     default:
 113         RELEASE_ASSERT_NOT_REACHED();
 114         return InvalidGPRReg;
 115     }
 116 }
 117
 118 bool SpeculativeJIT::fillJSValue(Edge edge, GPRReg& tagGPR, GPRReg& payloadGPR, FPRReg& fpr)
 119 {
 120     // FIXME: For double we could fill with a FPR.
 121     UNUSED_PARAM(fpr);
 122
 123     VirtualRegister virtualRegister = edge->virtualRegister();
 124     GenerationInfo& info = m_generationInfo[virtualRegister];
 125
 126     switch (info.registerFormat()) {
 127     case DataFormatNone: {
 128
 129         if (edge->hasConstant()) {
 130             tagGPR = allocate();
 131             payloadGPR = allocate();
 132             m_jit.move(Imm32(valueOfJSConstant(edge.node()).tag()), tagGPR);
 133             m_jit.move(Imm32(valueOfJSConstant(edge.node()).payload()), payloadGPR);
 134             m_gprs.retain(tagGPR, virtualRegister, SpillOrderConstant);
 135             m_gprs.retain(payloadGPR, virtualRegister, SpillOrderConstant);
 136             info.fillJSValue(*m_stream, tagGPR, payloadGPR, isInt32Constant(edge.node()) ? DataFormatJSInteger : DataFormatJS);
 137         } else {
 138             DataFormat spillFormat = info.spillFormat();
 139             ASSERT(spillFormat != DataFormatNone && spillFormat != DataFormatStorage);
 140             tagGPR = allocate();
 141             payloadGPR = allocate();
 142             switch (spillFormat) {
 143             case DataFormatInteger:
 144                 m_jit.move(TrustedImm32(JSValue::Int32Tag), tagGPR);
 145                 spillFormat = DataFormatJSInteger; // This will be used as the new register format.
 146                 break;
 147             case DataFormatCell:
 148                 m_jit.move(TrustedImm32(JSValue::CellTag), tagGPR);
 149                 spillFormat = DataFormatJSCell; // This will be used as the new register format.
 150                 break;
 151             case DataFormatBoolean:
 152                 m_jit.move(TrustedImm32(JSValue::BooleanTag), tagGPR);
 153                 spillFormat = DataFormatJSBoolean; // This will be used as the new register format.
 154                 break;
 155             default:
 156                 m_jit.load32(JITCompiler::tagFor(virtualRegister), tagGPR);
 157                 break;
 158             }
 159             m_jit.load32(JITCompiler::payloadFor(virtualRegister), payloadGPR);
 160             m_gprs.retain(tagGPR, virtualRegister, SpillOrderSpilled);
 161             m_gprs.retain(payloadGPR, virtualRegister, SpillOrderSpilled);
 162             info.fillJSValue(*m_stream, tagGPR, payloadGPR, spillFormat == DataFormatJSDouble ? DataFormatJS : spillFormat);
 163         }
 164
 165         return true;
 166     }
 167
 168     case DataFormatInteger:
 169     case DataFormatCell:
 170     case DataFormatBoolean: {
 171         GPRReg gpr = info.gpr();
 172         // If the register has already been locked we need to take a copy.
 173         if (m_gprs.isLocked(gpr)) {
 174             payloadGPR = allocate();
 175             m_jit.move(gpr, payloadGPR);
 176         } else {
 177             payloadGPR = gpr;
 178             m_gprs.lock(gpr);
 179         }
 180         tagGPR = allocate();
 181         uint32_t tag = JSValue::EmptyValueTag;
 182         DataFormat fillFormat = DataFormatJS;
 183         switch (info.registerFormat()) {
 184         case DataFormatInteger:
 185             tag = JSValue::Int32Tag;
 186             fillFormat = DataFormatJSInteger;
 187             break;
 188         case DataFormatCell:
 189             tag = JSValue::CellTag;
 190             fillFormat = DataFormatJSCell;
 191             break;
 192         case DataFormatBoolean:
 193             tag = JSValue::BooleanTag;
 194             fillFormat = DataFormatJSBoolean;
 195             break;
 196         default:
 197             RELEASE_ASSERT_NOT_REACHED();
 198             break;
 199         }
 200         m_jit.move(TrustedImm32(tag), tagGPR);
 201         m_gprs.release(gpr);
 202         m_gprs.retain(tagGPR, virtualRegister, SpillOrderJS);
 203         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderJS);
 204         info.fillJSValue(*m_stream, tagGPR, payloadGPR, fillFormat);
 205         return true;
 206     }
 207
 208     case DataFormatJSDouble:
 209     case DataFormatDouble: {
 210         FPRReg oldFPR = info.fpr();
 211         m_fprs.lock(oldFPR);
 212         tagGPR = allocate();
 213         payloadGPR = allocate();
 214         boxDouble(oldFPR, tagGPR, payloadGPR);
 215         m_fprs.unlock(oldFPR);
 216         m_fprs.release(oldFPR);
 217         m_gprs.retain(tagGPR, virtualRegister, SpillOrderJS);
 218         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderJS);
 219         info.fillJSValue(*m_stream, tagGPR, payloadGPR, DataFormatJS);
 220         return true;
 221     }
 222
 223     case DataFormatJS:
 224     case DataFormatJSInteger:
 225     case DataFormatJSCell:
 226     case DataFormatJSBoolean: {
 227         tagGPR = info.tagGPR();
 228         payloadGPR = info.payloadGPR();
 229         m_gprs.lock(tagGPR);
 230         m_gprs.lock(payloadGPR);
 231         return true;
 232     }
 233
 234     case DataFormatStorage:
 235         // this type currently never occurs
 236         RELEASE_ASSERT_NOT_REACHED();
 237
 238     default:
 239         RELEASE_ASSERT_NOT_REACHED();
 240         return true;
 241     }
 242 }
 243
 244 void SpeculativeJIT::nonSpeculativeUInt32ToNumber(Node* node)
 245 {
 246     IntegerOperand op1(this, node->child1());
 247     FPRTemporary boxer(this);
 248     GPRTemporary resultTag(this, op1);
 249     GPRTemporary resultPayload(this);
 250
 251     JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, op1.gpr(), TrustedImm32(0));
 252
 253     m_jit.convertInt32ToDouble(op1.gpr(), boxer.fpr());
 254     m_jit.move(JITCompiler::TrustedImmPtr(&AssemblyHelpers::twoToThe32), resultPayload.gpr()); // reuse resultPayload register here.
 255     m_jit.addDouble(JITCompiler::Address(resultPayload.gpr(), 0), boxer.fpr());
 256
 257     boxDouble(boxer.fpr(), resultTag.gpr(), resultPayload.gpr());
 258
 259     JITCompiler::Jump done = m_jit.jump();
 260
 261     positive.link(&m_jit);
 262
 263     m_jit.move(TrustedImm32(JSValue::Int32Tag), resultTag.gpr());
 264     m_jit.move(op1.gpr(), resultPayload.gpr());
 265
 266     done.link(&m_jit);
 267
 268     jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
 269 }
 270
 271 void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseTagGPROrNone, GPRReg basePayloadGPR, GPRReg resultTagGPR, GPRReg resultPayloadGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode)
 272 {
 273     JITCompiler::DataLabelPtr structureToCompare;
 274     JITCompiler::PatchableJump structureCheck = m_jit.patchableBranchPtrWithPatch(JITCompiler::NotEqual, JITCompiler::Address(basePayloadGPR, JSCell::structureOffset()), structureToCompare, JITCompiler::TrustedImmPtr(reinterpret_cast<void*>(unusedPointer)));
 275
 276     JITCompiler::ConvertibleLoadLabel propertyStorageLoad = m_jit.convertibleLoadPtr(JITCompiler::Address(basePayloadGPR, JSObject::butterflyOffset()), resultPayloadGPR);
 277     JITCompiler::DataLabelCompact tagLoadWithPatch = m_jit.load32WithCompactAddressOffsetPatch(JITCompiler::Address(resultPayloadGPR, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTagGPR);
 278     JITCompiler::DataLabelCompact payloadLoadWithPatch = m_jit.load32WithCompactAddressOffsetPatch(JITCompiler::Address(resultPayloadGPR, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR);
 279
 280     JITCompiler::Label doneLabel = m_jit.label();
 281
 282     OwnPtr<SlowPathGenerator> slowPath;
 283     if (baseTagGPROrNone == InvalidGPRReg) {
 284         if (!slowPathTarget.isSet()) {
 285             slowPath = slowPathCall(
 286                 structureCheck.m_jump, this, operationGetByIdOptimize,
 287                 JSValueRegs(resultTagGPR, resultPayloadGPR),
 288                 static_cast<int32_t>(JSValue::CellTag), basePayloadGPR,
 289                 identifier(identifierNumber));
 290         } else {
 291             JITCompiler::JumpList slowCases;
 292             slowCases.append(structureCheck.m_jump);
 293             slowCases.append(slowPathTarget);
 294             slowPath = slowPathCall(
 295                 slowCases, this, operationGetByIdOptimize,
 296                 JSValueRegs(resultTagGPR, resultPayloadGPR),
 297                 static_cast<int32_t>(JSValue::CellTag), basePayloadGPR,
 298                 identifier(identifierNumber));
 299         }
 300     } else {
 301         if (!slowPathTarget.isSet()) {
 302             slowPath = slowPathCall(
 303                 structureCheck.m_jump, this, operationGetByIdOptimize,
 304                 JSValueRegs(resultTagGPR, resultPayloadGPR), baseTagGPROrNone, basePayloadGPR,
 305                 identifier(identifierNumber));
 306         } else {
 307             JITCompiler::JumpList slowCases;
 308             slowCases.append(structureCheck.m_jump);
 309             slowCases.append(slowPathTarget);
 310             slowPath = slowPathCall(
 311                 slowCases, this, operationGetByIdOptimize,
 312                 JSValueRegs(resultTagGPR, resultPayloadGPR), baseTagGPROrNone, basePayloadGPR,
 313                 identifier(identifierNumber));
 314         }
 315     }
 316     m_jit.addPropertyAccess(
 317         PropertyAccessRecord(
 318             codeOrigin, structureToCompare, structureCheck, propertyStorageLoad,
 319             tagLoadWithPatch, payloadLoadWithPatch, slowPath.get(), doneLabel,
 320             safeCast<int8_t>(basePayloadGPR), safeCast<int8_t>(resultTagGPR),
 321             safeCast<int8_t>(resultPayloadGPR), usedRegisters(),
 322             spillMode == NeedToSpill ? PropertyAccessRecord::RegistersInUse : PropertyAccessRecord::RegistersFlushed));
 323     addSlowPathGenerator(slowPath.release());
 324 }
 325
 326 void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg basePayloadGPR, GPRReg valueTagGPR, GPRReg valuePayloadGPR, Edge valueUse, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget)
 327 {
 328     JITCompiler::DataLabelPtr structureToCompare;
 329     JITCompiler::PatchableJump structureCheck = m_jit.patchableBranchPtrWithPatch(JITCompiler::NotEqual, JITCompiler::Address(basePayloadGPR, JSCell::structureOffset()), structureToCompare, JITCompiler::TrustedImmPtr(reinterpret_cast<void*>(unusedPointer)));
 330
 331     writeBarrier(basePayloadGPR, valueTagGPR, valueUse, WriteBarrierForPropertyAccess, scratchGPR);
 332
 333     JITCompiler::ConvertibleLoadLabel propertyStorageLoad = m_jit.convertibleLoadPtr(JITCompiler::Address(basePayloadGPR, JSObject::butterflyOffset()), scratchGPR);
 334     JITCompiler::DataLabel32 tagStoreWithPatch = m_jit.store32WithAddressOffsetPatch(valueTagGPR, JITCompiler::Address(scratchGPR, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
 335     JITCompiler::DataLabel32 payloadStoreWithPatch = m_jit.store32WithAddressOffsetPatch(valuePayloadGPR, JITCompiler::Address(scratchGPR, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
 336
 337     JITCompiler::Label doneLabel = m_jit.label();
 338     V_DFGOperation_EJCI optimizedCall;
 339     if (m_jit.strictModeFor(m_currentNode->codeOrigin)) {
 340         if (putKind == Direct)
 341             optimizedCall = operationPutByIdDirectStrictOptimize;
 342         else
 343             optimizedCall = operationPutByIdStrictOptimize;
 344     } else {
 345         if (putKind == Direct)
 346             optimizedCall = operationPutByIdDirectNonStrictOptimize;
 347         else
 348             optimizedCall = operationPutByIdNonStrictOptimize;
 349     }
 350     OwnPtr<SlowPathGenerator> slowPath;
 351     if (!slowPathTarget.isSet()) {
 352         slowPath = slowPathCall(
 353             structureCheck.m_jump, this, optimizedCall, NoResult, valueTagGPR, valuePayloadGPR,
 354             basePayloadGPR, identifier(identifierNumber));
 355     } else {
 356         JITCompiler::JumpList slowCases;
 357         slowCases.append(structureCheck.m_jump);
 358         slowCases.append(slowPathTarget);
 359         slowPath = slowPathCall(
 360             slowCases, this, optimizedCall, NoResult, valueTagGPR, valuePayloadGPR,
 361             basePayloadGPR, identifier(identifierNumber));
 362     }
 363     RegisterSet currentlyUsedRegisters = usedRegisters();
 364     currentlyUsedRegisters.clear(scratchGPR);
 365     ASSERT(currentlyUsedRegisters.get(basePayloadGPR));
 366     ASSERT(currentlyUsedRegisters.get(valueTagGPR));
 367     ASSERT(currentlyUsedRegisters.get(valuePayloadGPR));
 368     m_jit.addPropertyAccess(
 369         PropertyAccessRecord(
 370             codeOrigin, structureToCompare, structureCheck, propertyStorageLoad,
 371             JITCompiler::DataLabelCompact(tagStoreWithPatch.label()),
 372             JITCompiler::DataLabelCompact(payloadStoreWithPatch.label()),
 373             slowPath.get(), doneLabel, safeCast<int8_t>(basePayloadGPR),
 374             safeCast<int8_t>(valueTagGPR), safeCast<int8_t>(valuePayloadGPR),
 375             usedRegisters()));
 376     addSlowPathGenerator(slowPath.release());
 377 }
 378
 379 void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool invert)
 380 {
 381     JSValueOperand arg(this, operand);
 382     GPRReg argTagGPR = arg.tagGPR();
 383     GPRReg argPayloadGPR = arg.payloadGPR();
 384
 385     GPRTemporary resultPayload(this, arg, false);
 386     GPRReg resultPayloadGPR = resultPayload.gpr();
 387
 388     JITCompiler::Jump notCell;
 389     JITCompiler::Jump notMasqueradesAsUndefined;
 390     if (m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
 391         if (!isKnownCell(operand.node()))
 392             notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag));
 393
 394         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
 395         m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultPayloadGPR);
 396         notMasqueradesAsUndefined = m_jit.jump();
 397     } else {
 398         GPRTemporary localGlobalObject(this);
 399         GPRTemporary remoteGlobalObject(this);
 400
 401         if (!isKnownCell(operand.node()))
 402             notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag));
 403
 404         m_jit.loadPtr(JITCompiler::Address(argPayloadGPR, JSCell::structureOffset()), resultPayloadGPR);
 405         JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(resultPayloadGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined));
 406
 407         m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultPayloadGPR);
 408         notMasqueradesAsUndefined = m_jit.jump();
 409
 410         isMasqueradesAsUndefined.link(&m_jit);
 411         GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
 412         GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
 413         m_jit.move(JITCompiler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)), localGlobalObjectGPR);
 414         m_jit.loadPtr(JITCompiler::Address(resultPayloadGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR);
 415         m_jit.compare32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultPayloadGPR);
 416     }
 417
 418     if (!isKnownCell(operand.node())) {
 419         JITCompiler::Jump done = m_jit.jump();
 420
 421         notCell.link(&m_jit);
 422         // null or undefined?
 423         COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag);
 424         m_jit.move(argTagGPR, resultPayloadGPR);
 425         m_jit.or32(TrustedImm32(1), resultPayloadGPR);
 426         m_jit.compare32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultPayloadGPR, TrustedImm32(JSValue::NullTag), resultPayloadGPR);
 427
 428         done.link(&m_jit);
 429     }
 430
 431     notMasqueradesAsUndefined.link(&m_jit);
 432
 433     booleanResult(resultPayloadGPR, m_currentNode);
 434 }
 435
 436 void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branchNode, bool invert)
 437 {
 438     BlockIndex taken = branchNode->takenBlockIndex();
 439     BlockIndex notTaken = branchNode->notTakenBlockIndex();
 440
 441     if (taken == nextBlock()) {
 442         invert = !invert;
 443         BlockIndex tmp = taken;
 444         taken = notTaken;
 445         notTaken = tmp;
 446     }
 447
 448     JSValueOperand arg(this, operand);
 449     GPRReg argTagGPR = arg.tagGPR();
 450     GPRReg argPayloadGPR = arg.payloadGPR();
 451
 452     GPRTemporary result(this, arg);
 453     GPRReg resultGPR = result.gpr();
 454
 455     JITCompiler::Jump notCell;
 456
 457     if (m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
 458         if (!isKnownCell(operand.node()))
 459             notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag));
 460
 461         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
 462         jump(invert ? taken : notTaken, ForceJump);
 463     } else {
 464         GPRTemporary localGlobalObject(this);
 465         GPRTemporary remoteGlobalObject(this);
 466
 467         if (!isKnownCell(operand.node()))
 468             notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag));
 469
 470         m_jit.loadPtr(JITCompiler::Address(argPayloadGPR, JSCell::structureOffset()), resultGPR);
 471         branchTest8(JITCompiler::Zero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined), invert ? taken : notTaken);
 472
 473         GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
 474         GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
 475         m_jit.move(TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)), localGlobalObjectGPR);
 476         m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR);
 477         branchPtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, invert ? notTaken : taken);
 478     }
 479
 480     if (!isKnownCell(operand.node())) {
 481         jump(notTaken, ForceJump);
 482
 483         notCell.link(&m_jit);
 484         // null or undefined?
 485         COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag);
 486         m_jit.move(argTagGPR, resultGPR);
 487         m_jit.or32(TrustedImm32(1), resultGPR);
 488         branch32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm32(JSValue::NullTag), taken);
 489     }
 490
 491     jump(notTaken);
 492 }
 493
 494 bool SpeculativeJIT::nonSpeculativeCompareNull(Node* node, Edge operand, bool invert)
 495 {
 496     unsigned branchIndexInBlock = detectPeepHoleBranch();
 497     if (branchIndexInBlock != UINT_MAX) {
 498         Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock);
 499
 500         ASSERT(node->adjustedRefCount() == 1);
 501
 502         nonSpeculativePeepholeBranchNull(operand, branchNode, invert);
 503
 504         use(node->child1());
 505         use(node->child2());
 506         m_indexInBlock = branchIndexInBlock;
 507         m_currentNode = branchNode;
 508
 509         return true;
 510     }
 511
 512     nonSpeculativeNonPeepholeCompareNull(operand, invert);
 513
 514     return false;
 515 }
 516
 517 void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction)
 518 {
 519     BlockIndex taken = branchNode->takenBlockIndex();
 520     BlockIndex notTaken = branchNode->notTakenBlockIndex();
 521
 522     JITCompiler::ResultCondition callResultCondition = JITCompiler::NonZero;
 523
 524     // The branch instruction will branch to the taken block.
 525     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
 526     if (taken == nextBlock()) {
 527         cond = JITCompiler::invert(cond);
 528         callResultCondition = JITCompiler::Zero;
 529         BlockIndex tmp = taken;
 530         taken = notTaken;
 531         notTaken = tmp;
 532     }
 533
 534     JSValueOperand arg1(this, node->child1());
 535     JSValueOperand arg2(this, node->child2());
 536     GPRReg arg1TagGPR = arg1.tagGPR();
 537     GPRReg arg1PayloadGPR = arg1.payloadGPR();
 538     GPRReg arg2TagGPR = arg2.tagGPR();
 539     GPRReg arg2PayloadGPR = arg2.payloadGPR();
 540
 541     JITCompiler::JumpList slowPath;
 542
 543     if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
 544         GPRResult result(this);
 545         GPRReg resultGPR = result.gpr();
 546
 547         arg1.use();
 548         arg2.use();
 549
 550         flushRegisters();
 551         callOperation(helperFunction, resultGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 552
 553         branchTest32(callResultCondition, resultGPR, taken);
 554     } else {
 555         GPRTemporary result(this);
 556         GPRReg resultGPR = result.gpr();
 557
 558         arg1.use();
 559         arg2.use();
 560
 561         if (!isKnownInteger(node->child1().node()))
 562             slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, arg1TagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)));
 563         if (!isKnownInteger(node->child2().node()))
 564             slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, arg2TagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)));
 565
 566         branch32(cond, arg1PayloadGPR, arg2PayloadGPR, taken);
 567
 568         if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
 569             jump(notTaken, ForceJump);
 570
 571             slowPath.link(&m_jit);
 572
 573             silentSpillAllRegisters(resultGPR);
 574             callOperation(helperFunction, resultGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 575             silentFillAllRegisters(resultGPR);
 576
 577             branchTest32(callResultCondition, resultGPR, taken);
 578         }
 579     }
 580
 581     jump(notTaken);
 582
 583     m_indexInBlock = m_jit.graph().m_blocks[m_block]->size() - 1;
 584     m_currentNode = branchNode;
 585 }
 586
 587 template<typename JumpType>
 588 class CompareAndBoxBooleanSlowPathGenerator
 589     : public CallSlowPathGenerator<JumpType, S_DFGOperation_EJJ, GPRReg> {
 590 public:
 591     CompareAndBoxBooleanSlowPathGenerator(
 592         JumpType from, SpeculativeJIT* jit,
 593         S_DFGOperation_EJJ function, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload,
 594         GPRReg arg2Tag, GPRReg arg2Payload)
 595         : CallSlowPathGenerator<JumpType, S_DFGOperation_EJJ, GPRReg>(
 596             from, jit, function, NeedToSpill, result)
 597         , m_arg1Tag(arg1Tag)
 598         , m_arg1Payload(arg1Payload)
 599         , m_arg2Tag(arg2Tag)
 600         , m_arg2Payload(arg2Payload)
 601     {
 602     }
 603
 604 protected:
 605     virtual void generateInternal(SpeculativeJIT* jit)
 606     {
 607         this->setUp(jit);
 608         this->recordCall(
 609             jit->callOperation(
 610                 this->m_function, this->m_result, m_arg1Tag, m_arg1Payload, m_arg2Tag,
 611                 m_arg2Payload));
 612         jit->m_jit.and32(JITCompiler::TrustedImm32(1), this->m_result);
 613         this->tearDown(jit);
 614     }
 615
 616 private:
 617     GPRReg m_arg1Tag;
 618     GPRReg m_arg1Payload;
 619     GPRReg m_arg2Tag;
 620     GPRReg m_arg2Payload;
 621 };
 622
 623 void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction)
 624 {
 625     JSValueOperand arg1(this, node->child1());
 626     JSValueOperand arg2(this, node->child2());
 627     GPRReg arg1TagGPR = arg1.tagGPR();
 628     GPRReg arg1PayloadGPR = arg1.payloadGPR();
 629     GPRReg arg2TagGPR = arg2.tagGPR();
 630     GPRReg arg2PayloadGPR = arg2.payloadGPR();
 631
 632     JITCompiler::JumpList slowPath;
 633
 634     if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) {
 635         GPRResult result(this);
 636         GPRReg resultPayloadGPR = result.gpr();
 637
 638         arg1.use();
 639         arg2.use();
 640
 641         flushRegisters();
 642         callOperation(helperFunction, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 643
 644         booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly);
 645     } else {
 646         GPRTemporary resultPayload(this, arg1, false);
 647         GPRReg resultPayloadGPR = resultPayload.gpr();
 648
 649         arg1.use();
 650         arg2.use();
 651
 652         if (!isKnownInteger(node->child1().node()))
 653             slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, arg1TagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)));
 654         if (!isKnownInteger(node->child2().node()))
 655             slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, arg2TagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)));
 656
 657         m_jit.compare32(cond, arg1PayloadGPR, arg2PayloadGPR, resultPayloadGPR);
 658
 659         if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) {
 660             addSlowPathGenerator(adoptPtr(
 661                 new CompareAndBoxBooleanSlowPathGenerator<JITCompiler::JumpList>(
 662                     slowPath, this, helperFunction, resultPayloadGPR, arg1TagGPR,
 663                     arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR)));
 664         }
 665
 666         booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly);
 667     }
 668 }
 669
 670 void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode, bool invert)
 671 {
 672     BlockIndex taken = branchNode->takenBlockIndex();
 673     BlockIndex notTaken = branchNode->notTakenBlockIndex();
 674
 675     // The branch instruction will branch to the taken block.
 676     // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through.
 677     if (taken == nextBlock()) {
 678         invert = !invert;
 679         BlockIndex tmp = taken;
 680         taken = notTaken;
 681         notTaken = tmp;
 682     }
 683
 684     JSValueOperand arg1(this, node->child1());
 685     JSValueOperand arg2(this, node->child2());
 686     GPRReg arg1TagGPR = arg1.tagGPR();
 687     GPRReg arg1PayloadGPR = arg1.payloadGPR();
 688     GPRReg arg2TagGPR = arg2.tagGPR();
 689     GPRReg arg2PayloadGPR = arg2.payloadGPR();
 690
 691     GPRTemporary resultPayload(this, arg1, false);
 692     GPRReg resultPayloadGPR = resultPayload.gpr();
 693
 694     arg1.use();
 695     arg2.use();
 696
 697     if (isKnownCell(node->child1().node()) && isKnownCell(node->child2().node())) {
 698         // see if we get lucky: if the arguments are cells and they reference the same
 699         // cell, then they must be strictly equal.
 700         branchPtr(JITCompiler::Equal, arg1PayloadGPR, arg2PayloadGPR, invert ? notTaken : taken);
 701
 702         silentSpillAllRegisters(resultPayloadGPR);
 703         callOperation(operationCompareStrictEqCell, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 704         silentFillAllRegisters(resultPayloadGPR);
 705
 706         branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultPayloadGPR, taken);
 707     } else {
 708         // FIXME: Add fast paths for twoCells, number etc.
 709
 710         silentSpillAllRegisters(resultPayloadGPR);
 711         callOperation(operationCompareStrictEq, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 712         silentFillAllRegisters(resultPayloadGPR);
 713
 714         branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultPayloadGPR, taken);
 715     }
 716
 717     jump(notTaken);
 718 }
 719
 720 void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert)
 721 {
 722     JSValueOperand arg1(this, node->child1());
 723     JSValueOperand arg2(this, node->child2());
 724     GPRReg arg1TagGPR = arg1.tagGPR();
 725     GPRReg arg1PayloadGPR = arg1.payloadGPR();
 726     GPRReg arg2TagGPR = arg2.tagGPR();
 727     GPRReg arg2PayloadGPR = arg2.payloadGPR();
 728
 729     GPRTemporary resultPayload(this, arg1, false);
 730     GPRReg resultPayloadGPR = resultPayload.gpr();
 731
 732     arg1.use();
 733     arg2.use();
 734
 735     if (isKnownCell(node->child1().node()) && isKnownCell(node->child2().node())) {
 736         // see if we get lucky: if the arguments are cells and they reference the same
 737         // cell, then they must be strictly equal.
 738         // FIXME: this should flush registers instead of silent spill/fill.
 739         JITCompiler::Jump notEqualCase = m_jit.branchPtr(JITCompiler::NotEqual, arg1PayloadGPR, arg2PayloadGPR);
 740
 741         m_jit.move(JITCompiler::TrustedImm32(!invert), resultPayloadGPR);
 742         JITCompiler::Jump done = m_jit.jump();
 743
 744         notEqualCase.link(&m_jit);
 745
 746         silentSpillAllRegisters(resultPayloadGPR);
 747         callOperation(operationCompareStrictEqCell, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 748         silentFillAllRegisters(resultPayloadGPR);
 749
 750         m_jit.andPtr(JITCompiler::TrustedImm32(1), resultPayloadGPR);
 751
 752         done.link(&m_jit);
 753     } else {
 754         // FIXME: Add fast paths.
 755
 756         silentSpillAllRegisters(resultPayloadGPR);
 757         callOperation(operationCompareStrictEq, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR);
 758         silentFillAllRegisters(resultPayloadGPR);
 759
 760         m_jit.andPtr(JITCompiler::TrustedImm32(1), resultPayloadGPR);
 761     }
 762
 763     booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly);
 764 }
 765
 766 void SpeculativeJIT::emitCall(Node* node)
 767 {
 768     if (node->op() != Call)
 769         ASSERT(node->op() == Construct);
 770
 771     // For constructors, the this argument is not passed but we have to make space
 772     // for it.
 773     int dummyThisArgument = node->op() == Call ? 0 : 1;
 774
 775     CallLinkInfo::CallType callType = node->op() == Call ? CallLinkInfo::Call : CallLinkInfo::Construct;
 776
 777     Edge calleeEdge = m_jit.graph().m_varArgChildren[node->firstChild()];
 778     JSValueOperand callee(this, calleeEdge);
 779     GPRReg calleeTagGPR = callee.tagGPR();
 780     GPRReg calleePayloadGPR = callee.payloadGPR();
 781     use(calleeEdge);
 782
 783     // The call instruction's first child is either the function (normal call) or the
 784     // receiver (method call). subsequent children are the arguments.
 785     int numPassedArgs = node->numChildren() - 1;
 786
 787     m_jit.store32(MacroAssembler::TrustedImm32(numPassedArgs + dummyThisArgument), callFramePayloadSlot(JSStack::ArgumentCount));
 788     m_jit.storePtr(GPRInfo::callFrameRegister, callFramePayloadSlot(JSStack::CallerFrame));
 789     m_jit.store32(calleePayloadGPR, callFramePayloadSlot(JSStack::Callee));
 790     m_jit.store32(calleeTagGPR, callFrameTagSlot(JSStack::Callee));
 791
 792     for (int i = 0; i < numPassedArgs; i++) {
 793         Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i];
 794         JSValueOperand arg(this, argEdge);
 795         GPRReg argTagGPR = arg.tagGPR();
 796         GPRReg argPayloadGPR = arg.payloadGPR();
 797         use(argEdge);
 798
 799         m_jit.store32(argTagGPR, argumentTagSlot(i + dummyThisArgument));
 800         m_jit.store32(argPayloadGPR, argumentPayloadSlot(i + dummyThisArgument));
 801     }
 802
 803     flushRegisters();
 804
 805     GPRResult resultPayload(this);
 806     GPRResult2 resultTag(this);
 807     GPRReg resultPayloadGPR = resultPayload.gpr();
 808     GPRReg resultTagGPR = resultTag.gpr();
 809
 810     JITCompiler::DataLabelPtr targetToCheck;
 811     JITCompiler::JumpList slowPath;
 812
 813     CallBeginToken token;
 814     m_jit.beginCall(node->codeOrigin, token);
 815
 816     m_jit.addPtr(TrustedImm32(m_jit.codeBlock()->m_numCalleeRegisters * sizeof(Register)), GPRInfo::callFrameRegister);
 817
 818     slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, calleeTagGPR, TrustedImm32(JSValue::CellTag)));
 819     slowPath.append(m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleePayloadGPR, targetToCheck));
 820     m_jit.loadPtr(MacroAssembler::Address(calleePayloadGPR, OBJECT_OFFSETOF(JSFunction, m_scope)), resultPayloadGPR);
 821     m_jit.storePtr(resultPayloadGPR, MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
 822     m_jit.store32(MacroAssembler::TrustedImm32(JSValue::CellTag), MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
 823
 824     CodeOrigin codeOrigin = node->codeOrigin;
 825     JITCompiler::Call fastCall = m_jit.nearCall();
 826     m_jit.notifyCall(fastCall, codeOrigin, token);
 827
 828     JITCompiler::Jump done = m_jit.jump();
 829
 830     slowPath.link(&m_jit);
 831
 832     if (calleeTagGPR == GPRInfo::nonArgGPR0) {
 833         if (calleePayloadGPR == GPRInfo::nonArgGPR1)
 834             m_jit.swap(GPRInfo::nonArgGPR1, GPRInfo::nonArgGPR0);
 835         else {
 836             m_jit.move(calleeTagGPR, GPRInfo::nonArgGPR1);
 837             m_jit.move(calleePayloadGPR, GPRInfo::nonArgGPR0);
 838         }
 839     } else {
 840         m_jit.move(calleePayloadGPR, GPRInfo::nonArgGPR0);
 841         m_jit.move(calleeTagGPR, GPRInfo::nonArgGPR1);
 842     }
 843     m_jit.prepareForExceptionCheck();
 844     JITCompiler::Call slowCall = m_jit.nearCall();
 845     m_jit.notifyCall(slowCall, codeOrigin, token);
 846
 847     done.link(&m_jit);
 848
 849     m_jit.setupResults(resultPayloadGPR, resultTagGPR);
 850
 851     jsValueResult(resultTagGPR, resultPayloadGPR, node, DataFormatJS, UseChildrenCalledExplicitly);
 852
 853     m_jit.addJSCall(fastCall, slowCall, targetToCheck, callType, calleePayloadGPR, node->codeOrigin);
 854 }
 855
 856 template<bool strict>
 857 GPRReg SpeculativeJIT::fillSpeculateIntInternal(Edge edge, DataFormat& returnFormat)
 858 {
 859 #if DFG_ENABLE(DEBUG_VERBOSE)
 860     dataLogF("SpecInt@%d   ", edge->index());
 861 #endif
 862     AbstractValue& value = m_state.forNode(edge);
 863     SpeculatedType type = value.m_type;
 864     ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32));
 865     value.filter(SpecInt32);
 866     VirtualRegister virtualRegister = edge->virtualRegister();
 867     GenerationInfo& info = m_generationInfo[virtualRegister];
 868
 869     switch (info.registerFormat()) {
 870     case DataFormatNone: {
 871         if ((edge->hasConstant() && !isInt32Constant(edge.node())) || info.spillFormat() == DataFormatDouble) {
 872             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 873             returnFormat = DataFormatInteger;
 874             return allocate();
 875         }
 876
 877         if (edge->hasConstant()) {
 878             ASSERT(isInt32Constant(edge.node()));
 879             GPRReg gpr = allocate();
 880             m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
 881             m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
 882             info.fillInteger(*m_stream, gpr);
 883             returnFormat = DataFormatInteger;
 884             return gpr;
 885         }
 886
 887         DataFormat spillFormat = info.spillFormat();
 888         ASSERT_UNUSED(spillFormat, (spillFormat & DataFormatJS) || spillFormat == DataFormatInteger);
 889
 890         // If we know this was spilled as an integer we can fill without checking.
 891         if (type & ~SpecInt32)
 892             speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)));
 893
 894         GPRReg gpr = allocate();
 895         m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr);
 896         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
 897         info.fillInteger(*m_stream, gpr);
 898         returnFormat = DataFormatInteger;
 899         return gpr;
 900     }
 901
 902     case DataFormatJSInteger:
 903     case DataFormatJS: {
 904         // Check the value is an integer.
 905         GPRReg tagGPR = info.tagGPR();
 906         GPRReg payloadGPR = info.payloadGPR();
 907         m_gprs.lock(tagGPR);
 908         m_gprs.lock(payloadGPR);
 909         if (type & ~SpecInt32)
 910             speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::Int32Tag)));
 911         m_gprs.unlock(tagGPR);
 912         m_gprs.release(tagGPR);
 913         m_gprs.release(payloadGPR);
 914         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderInteger);
 915         info.fillInteger(*m_stream, payloadGPR);
 916         // If !strict we're done, return.
 917         returnFormat = DataFormatInteger;
 918         return payloadGPR;
 919     }
 920
 921     case DataFormatInteger: {
 922         GPRReg gpr = info.gpr();
 923         m_gprs.lock(gpr);
 924         returnFormat = DataFormatInteger;
 925         return gpr;
 926     }
 927
 928     case DataFormatDouble:
 929     case DataFormatCell:
 930     case DataFormatBoolean:
 931     case DataFormatJSDouble:
 932     case DataFormatJSCell:
 933     case DataFormatJSBoolean:
 934         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 935         returnFormat = DataFormatInteger;
 936         return allocate();
 937
 938     case DataFormatStorage:
 939     default:
 940         RELEASE_ASSERT_NOT_REACHED();
 941         return InvalidGPRReg;
 942     }
 943 }
 944
 945 GPRReg SpeculativeJIT::fillSpeculateInt(Edge edge, DataFormat& returnFormat)
 946 {
 947     return fillSpeculateIntInternal<false>(edge, returnFormat);
 948 }
 949
 950 GPRReg SpeculativeJIT::fillSpeculateIntStrict(Edge edge)
 951 {
 952     DataFormat mustBeDataFormatInteger;
 953     GPRReg result = fillSpeculateIntInternal<true>(edge, mustBeDataFormatInteger);
 954     ASSERT(mustBeDataFormatInteger == DataFormatInteger);
 955     return result;
 956 }
 957
 958 FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge)
 959 {
 960 #if DFG_ENABLE(DEBUG_VERBOSE)
 961     dataLogF("SpecDouble@%d   ", edge->index());
 962 #endif
 963     AbstractValue& value = m_state.forNode(edge);
 964     SpeculatedType type = value.m_type;
 965     ASSERT(edge.useKind() != KnownNumberUse || !(value.m_type & ~SpecNumber));
 966     value.filter(SpecNumber);
 967     VirtualRegister virtualRegister = edge->virtualRegister();
 968     GenerationInfo& info = m_generationInfo[virtualRegister];
 969
 970     if (info.registerFormat() == DataFormatNone) {
 971
 972         if (edge->hasConstant()) {
 973             if (isInt32Constant(edge.node())) {
 974                 GPRReg gpr = allocate();
 975                 m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr);
 976                 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
 977                 info.fillInteger(*m_stream, gpr);
 978                 unlock(gpr);
 979             } else if (isNumberConstant(edge.node())) {
 980                 FPRReg fpr = fprAllocate();
 981                 m_jit.loadDouble(addressOfDoubleConstant(edge.node()), fpr);
 982                 m_fprs.retain(fpr, virtualRegister, SpillOrderConstant);
 983                 info.fillDouble(*m_stream, fpr);
 984                 return fpr;
 985             } else {
 986                 terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
 987                 return fprAllocate();
 988             }
 989         } else {
 990             DataFormat spillFormat = info.spillFormat();
 991             ASSERT((spillFormat & DataFormatJS) || spillFormat == DataFormatInteger);
 992             if (spillFormat == DataFormatJSDouble || spillFormat == DataFormatDouble) {
 993                 FPRReg fpr = fprAllocate();
 994                 m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr);
 995                 m_fprs.retain(fpr, virtualRegister, SpillOrderSpilled);
 996                 info.fillDouble(*m_stream, fpr);
 997                 return fpr;
 998             }
 999
1000             FPRReg fpr = fprAllocate();
1001             JITCompiler::Jump hasUnboxedDouble;
1002
1003             if (spillFormat != DataFormatJSInteger && spillFormat != DataFormatInteger) {
1004                 JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag));
1005                 if (type & ~SpecNumber)
1006                     speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::AboveOrEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::LowestTag)));
1007                 m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr);
1008                 hasUnboxedDouble = m_jit.jump();
1009
1010                 isInteger.link(&m_jit);
1011             }
1012
1013             m_jit.convertInt32ToDouble(JITCompiler::payloadFor(virtualRegister), fpr);
1014
1015             if (hasUnboxedDouble.isSet())
1016                 hasUnboxedDouble.link(&m_jit);
1017
1018             m_fprs.retain(fpr, virtualRegister, SpillOrderSpilled);
1019             info.fillDouble(*m_stream, fpr);
1020             info.killSpilled();
1021             return fpr;
1022         }
1023     }
1024
1025     switch (info.registerFormat()) {
1026     case DataFormatJS:
1027     case DataFormatJSInteger: {
1028         GPRReg tagGPR = info.tagGPR();
1029         GPRReg payloadGPR = info.payloadGPR();
1030         FPRReg fpr = fprAllocate();
1031
1032         m_gprs.lock(tagGPR);
1033         m_gprs.lock(payloadGPR);
1034
1035         JITCompiler::Jump hasUnboxedDouble;
1036
1037         if (info.registerFormat() != DataFormatJSInteger) {
1038             FPRTemporary scratch(this);
1039             JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag));
1040             if (type & ~SpecNumber)
1041                 speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::AboveOrEqual, tagGPR, TrustedImm32(JSValue::LowestTag)));
1042             unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr());
1043             hasUnboxedDouble = m_jit.jump();
1044             isInteger.link(&m_jit);
1045         }
1046
1047         m_jit.convertInt32ToDouble(payloadGPR, fpr);
1048
1049         if (hasUnboxedDouble.isSet())
1050             hasUnboxedDouble.link(&m_jit);
1051
1052         m_gprs.release(tagGPR);
1053         m_gprs.release(payloadGPR);
1054         m_gprs.unlock(tagGPR);
1055         m_gprs.unlock(payloadGPR);
1056         m_fprs.retain(fpr, virtualRegister, SpillOrderDouble);
1057         info.fillDouble(*m_stream, fpr);
1058         info.killSpilled();
1059         return fpr;
1060     }
1061
1062     case DataFormatInteger: {
1063         FPRReg fpr = fprAllocate();
1064         GPRReg gpr = info.gpr();
1065         m_gprs.lock(gpr);
1066         m_jit.convertInt32ToDouble(gpr, fpr);
1067         m_gprs.unlock(gpr);
1068         return fpr;
1069     }
1070
1071     case DataFormatJSDouble:
1072     case DataFormatDouble: {
1073         FPRReg fpr = info.fpr();
1074         m_fprs.lock(fpr);
1075         return fpr;
1076     }
1077
1078     case DataFormatNone:
1079     case DataFormatStorage:
1080         RELEASE_ASSERT_NOT_REACHED();
1081
1082     case DataFormatCell:
1083     case DataFormatJSCell:
1084     case DataFormatBoolean:
1085     case DataFormatJSBoolean:
1086         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1087         return fprAllocate();
1088
1089     default:
1090         RELEASE_ASSERT_NOT_REACHED();
1091         return InvalidFPRReg;
1092     }
1093 }
1094
1095 GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge)
1096 {
1097 #if DFG_ENABLE(DEBUG_VERBOSE)
1098     dataLogF("SpecCell@%d   ", edge->index());
1099 #endif
1100     AbstractValue& value = m_state.forNode(edge);
1101     SpeculatedType type = value.m_type;
1102     ASSERT((edge.useKind() != KnownCellUse && edge.useKind() != KnownStringUse) || !(value.m_type & ~SpecCell));
1103     value.filter(SpecCell);
1104     VirtualRegister virtualRegister = edge->virtualRegister();
1105     GenerationInfo& info = m_generationInfo[virtualRegister];
1106
1107     switch (info.registerFormat()) {
1108     case DataFormatNone: {
1109         if (info.spillFormat() == DataFormatInteger || info.spillFormat() == DataFormatDouble) {
1110             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1111             return allocate();
1112         }
1113
1114         if (edge->hasConstant()) {
1115             JSValue jsValue = valueOfJSConstant(edge.node());
1116             GPRReg gpr = allocate();
1117             if (jsValue.isCell()) {
1118                 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
1119                 m_jit.move(MacroAssembler::TrustedImmPtr(jsValue.asCell()), gpr);
1120                 info.fillCell(*m_stream, gpr);
1121                 return gpr;
1122             }
1123             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1124             return gpr;
1125         }
1126
1127         ASSERT((info.spillFormat() & DataFormatJS) || info.spillFormat() == DataFormatCell);
1128         if (type & ~SpecCell)
1129             speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag)));
1130         GPRReg gpr = allocate();
1131         m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr);
1132         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
1133         info.fillCell(*m_stream, gpr);
1134         return gpr;
1135     }
1136
1137     case DataFormatCell: {
1138         GPRReg gpr = info.gpr();
1139         m_gprs.lock(gpr);
1140         return gpr;
1141     }
1142
1143     case DataFormatJSCell:
1144     case DataFormatJS: {
1145         GPRReg tagGPR = info.tagGPR();
1146         GPRReg payloadGPR = info.payloadGPR();
1147         m_gprs.lock(tagGPR);
1148         m_gprs.lock(payloadGPR);
1149         if (type & ~SpecCell)
1150             speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::CellTag)));
1151         m_gprs.unlock(tagGPR);
1152         m_gprs.release(tagGPR);
1153         m_gprs.release(payloadGPR);
1154         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderCell);
1155         info.fillCell(*m_stream, payloadGPR);
1156         return payloadGPR;
1157     }
1158
1159     case DataFormatJSInteger:
1160     case DataFormatInteger:
1161     case DataFormatJSDouble:
1162     case DataFormatDouble:
1163     case DataFormatJSBoolean:
1164     case DataFormatBoolean:
1165         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1166         return allocate();
1167
1168     case DataFormatStorage:
1169         RELEASE_ASSERT_NOT_REACHED();
1170
1171     default:
1172         RELEASE_ASSERT_NOT_REACHED();
1173         return InvalidGPRReg;
1174     }
1175 }
1176
1177 GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge)
1178 {
1179 #if DFG_ENABLE(DEBUG_VERBOSE)
1180     dataLogF("SpecBool@%d   ", edge.node()->index());
1181 #endif
1182     AbstractValue& value = m_state.forNode(edge);
1183     SpeculatedType type = value.m_type;
1184     value.filter(SpecBoolean);
1185     VirtualRegister virtualRegister = edge->virtualRegister();
1186     GenerationInfo& info = m_generationInfo[virtualRegister];
1187
1188     switch (info.registerFormat()) {
1189     case DataFormatNone: {
1190         if (info.spillFormat() == DataFormatInteger || info.spillFormat() == DataFormatDouble) {
1191             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1192             return allocate();
1193         }
1194
1195         if (edge->hasConstant()) {
1196             JSValue jsValue = valueOfJSConstant(edge.node());
1197             GPRReg gpr = allocate();
1198             if (jsValue.isBoolean()) {
1199                 m_gprs.retain(gpr, virtualRegister, SpillOrderConstant);
1200                 m_jit.move(MacroAssembler::TrustedImm32(jsValue.asBoolean()), gpr);
1201                 info.fillBoolean(*m_stream, gpr);
1202                 return gpr;
1203             }
1204             terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1205             return gpr;
1206         }
1207
1208         ASSERT((info.spillFormat() & DataFormatJS) || info.spillFormat() == DataFormatBoolean);
1209
1210         if (type & ~SpecBoolean)
1211             speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::BooleanTag)));
1212
1213         GPRReg gpr = allocate();
1214         m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr);
1215         m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled);
1216         info.fillBoolean(*m_stream, gpr);
1217         return gpr;
1218     }
1219
1220     case DataFormatBoolean: {
1221         GPRReg gpr = info.gpr();
1222         m_gprs.lock(gpr);
1223         return gpr;
1224     }
1225
1226     case DataFormatJSBoolean:
1227     case DataFormatJS: {
1228         GPRReg tagGPR = info.tagGPR();
1229         GPRReg payloadGPR = info.payloadGPR();
1230         m_gprs.lock(tagGPR);
1231         m_gprs.lock(payloadGPR);
1232         if (type & ~SpecBoolean)
1233             speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::BooleanTag)));
1234
1235         m_gprs.unlock(tagGPR);
1236         m_gprs.release(tagGPR);
1237         m_gprs.release(payloadGPR);
1238         m_gprs.retain(payloadGPR, virtualRegister, SpillOrderBoolean);
1239         info.fillBoolean(*m_stream, payloadGPR);
1240         return payloadGPR;
1241     }
1242
1243     case DataFormatJSInteger:
1244     case DataFormatInteger:
1245     case DataFormatJSDouble:
1246     case DataFormatDouble:
1247     case DataFormatJSCell:
1248     case DataFormatCell:
1249         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
1250         return allocate();
1251
1252     case DataFormatStorage:
1253         RELEASE_ASSERT_NOT_REACHED();
1254
1255     default:
1256         RELEASE_ASSERT_NOT_REACHED();
1257         return InvalidGPRReg;
1258     }
1259 }
1260
1261 JITCompiler::Jump SpeculativeJIT::convertToDouble(JSValueOperand& op, FPRReg result)
1262 {
1263     FPRTemporary scratch(this);
1264
1265     GPRReg opPayloadGPR = op.payloadGPR();
1266     GPRReg opTagGPR = op.tagGPR();
1267     FPRReg scratchFPR = scratch.fpr();
1268
1269     JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, opTagGPR, TrustedImm32(JSValue::Int32Tag));
1270     JITCompiler::Jump notNumber = m_jit.branch32(MacroAssembler::AboveOrEqual, opPayloadGPR, TrustedImm32(JSValue::LowestTag));
1271
1272     unboxDouble(opTagGPR, opPayloadGPR, result, scratchFPR);
1273     JITCompiler::Jump done = m_jit.jump();
1274
1275     isInteger.link(&m_jit);
1276     m_jit.convertInt32ToDouble(opPayloadGPR, result);
1277
1278     done.link(&m_jit);
1279
1280     return notNumber;
1281 }
1282
1283 void SpeculativeJIT::compileObjectEquality(Node* node)
1284 {
1285     SpeculateCellOperand op1(this, node->child1());
1286     SpeculateCellOperand op2(this, node->child2());
1287     GPRReg op1GPR = op1.gpr();
1288     GPRReg op2GPR = op2.gpr();
1289
1290     if (m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
1291         m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1292         DFG_TYPE_CHECK(
1293             JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr(
1294                 MacroAssembler::Equal,
1295                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1296                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1297         DFG_TYPE_CHECK(
1298             JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr(
1299                 MacroAssembler::Equal,
1300                 MacroAssembler::Address(op2GPR, JSCell::structureOffset()),
1301                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1302     } else {
1303         GPRTemporary structure(this);
1304         GPRReg structureGPR = structure.gpr();
1305
1306         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1307         DFG_TYPE_CHECK(
1308             JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr(
1309                 MacroAssembler::Equal,
1310                 structureGPR,
1311                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1312         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(),
1313             m_jit.branchTest8(
1314                 MacroAssembler::NonZero,
1315                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1316                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1317
1318         m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR);
1319         DFG_TYPE_CHECK(
1320             JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr(
1321                 MacroAssembler::Equal,
1322                 structureGPR,
1323                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1324         speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(),
1325             m_jit.branchTest8(
1326                 MacroAssembler::NonZero,
1327                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1328                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1329     }
1330
1331     GPRTemporary resultPayload(this, op2);
1332     GPRReg resultPayloadGPR = resultPayload.gpr();
1333
1334     MacroAssembler::Jump falseCase = m_jit.branchPtr(MacroAssembler::NotEqual, op1GPR, op2GPR);
1335     m_jit.move(TrustedImm32(1), resultPayloadGPR);
1336     MacroAssembler::Jump done = m_jit.jump();
1337     falseCase.link(&m_jit);
1338     m_jit.move(TrustedImm32(0), resultPayloadGPR);
1339     done.link(&m_jit);
1340
1341     booleanResult(resultPayloadGPR, node);
1342 }
1343
1344 void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild)
1345 {
1346     SpeculateCellOperand op1(this, leftChild);
1347     JSValueOperand op2(this, rightChild, ManualOperandSpeculation);
1348     GPRTemporary result(this);
1349
1350     GPRReg op1GPR = op1.gpr();
1351     GPRReg op2TagGPR = op2.tagGPR();
1352     GPRReg op2PayloadGPR = op2.payloadGPR();
1353     GPRReg resultGPR = result.gpr();
1354     GPRTemporary structure;
1355     GPRReg structureGPR = InvalidGPRReg;
1356
1357     bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid();
1358
1359     if (!masqueradesAsUndefinedWatchpointValid) {
1360         // The masquerades as undefined case will use the structure register, so allocate it here.
1361         // Do this at the top of the function to avoid branching around a register allocation.
1362         GPRTemporary realStructure(this);
1363         structure.adopt(realStructure);
1364         structureGPR = structure.gpr();
1365     }
1366
1367     if (masqueradesAsUndefinedWatchpointValid) {
1368         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1369         DFG_TYPE_CHECK(
1370             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1371                 MacroAssembler::Equal,
1372                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1373                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1374     } else {
1375         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1376         DFG_TYPE_CHECK(
1377             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1378                 MacroAssembler::Equal,
1379                 structureGPR,
1380                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1381         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild,
1382             m_jit.branchTest8(
1383                 MacroAssembler::NonZero,
1384                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1385                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1386     }
1387
1388
1389     // It seems that most of the time when programs do a == b where b may be either null/undefined
1390     // or an object, b is usually an object. Balance the branches to make that case fast.
1391     MacroAssembler::Jump rightNotCell =
1392         m_jit.branch32(MacroAssembler::NotEqual, op2TagGPR, TrustedImm32(JSValue::CellTag));
1393
1394     // We know that within this branch, rightChild must be a cell.
1395     if (masqueradesAsUndefinedWatchpointValid) {
1396         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1397         DFG_TYPE_CHECK(
1398             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject,
1399             m_jit.branchPtr(
1400                 MacroAssembler::Equal,
1401                 MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()),
1402                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1403     } else {
1404         m_jit.loadPtr(MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()), structureGPR);
1405         DFG_TYPE_CHECK(
1406             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject,
1407             m_jit.branchPtr(
1408                 MacroAssembler::Equal,
1409                 structureGPR,
1410                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1411         speculationCheck(BadType, JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild,
1412             m_jit.branchTest8(
1413                 MacroAssembler::NonZero,
1414                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1415                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1416     }
1417
1418     // At this point we know that we can perform a straight-forward equality comparison on pointer
1419     // values because both left and right are pointers to objects that have no special equality
1420     // protocols.
1421     MacroAssembler::Jump falseCase = m_jit.branchPtr(MacroAssembler::NotEqual, op1GPR, op2PayloadGPR);
1422     MacroAssembler::Jump trueCase = m_jit.jump();
1423
1424     rightNotCell.link(&m_jit);
1425
1426     // We know that within this branch, rightChild must not be a cell. Check if that is enough to
1427     // prove that it is either null or undefined.
1428     if (needsTypeCheck(rightChild, SpecCell | SpecOther)) {
1429         m_jit.move(op2TagGPR, resultGPR);
1430         m_jit.or32(TrustedImm32(1), resultGPR);
1431
1432         typeCheck(
1433             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, SpecCell | SpecOther,
1434             m_jit.branch32(
1435                 MacroAssembler::NotEqual, resultGPR,
1436                 MacroAssembler::TrustedImm32(JSValue::NullTag)));
1437     }
1438
1439     falseCase.link(&m_jit);
1440     m_jit.move(TrustedImm32(0), resultGPR);
1441     MacroAssembler::Jump done = m_jit.jump();
1442     trueCase.link(&m_jit);
1443     m_jit.move(TrustedImm32(1), resultGPR);
1444     done.link(&m_jit);
1445
1446     booleanResult(resultGPR, m_currentNode);
1447 }
1448
1449 void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild, Node* branchNode)
1450 {
1451     BlockIndex taken = branchNode->takenBlockIndex();
1452     BlockIndex notTaken = branchNode->notTakenBlockIndex();
1453
1454     SpeculateCellOperand op1(this, leftChild);
1455     JSValueOperand op2(this, rightChild, ManualOperandSpeculation);
1456     GPRTemporary result(this);
1457
1458     GPRReg op1GPR = op1.gpr();
1459     GPRReg op2TagGPR = op2.tagGPR();
1460     GPRReg op2PayloadGPR = op2.payloadGPR();
1461     GPRReg resultGPR = result.gpr();
1462     GPRTemporary structure;
1463     GPRReg structureGPR = InvalidGPRReg;
1464
1465     bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid();
1466
1467     if (!masqueradesAsUndefinedWatchpointValid) {
1468         // The masquerades as undefined case will use the structure register, so allocate it here.
1469         // Do this at the top of the function to avoid branching around a register allocation.
1470         GPRTemporary realStructure(this);
1471         structure.adopt(realStructure);
1472         structureGPR = structure.gpr();
1473     }
1474
1475     if (masqueradesAsUndefinedWatchpointValid) {
1476         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1477         DFG_TYPE_CHECK(
1478             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1479                 MacroAssembler::Equal,
1480                 MacroAssembler::Address(op1GPR, JSCell::structureOffset()),
1481                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1482     } else {
1483         m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR);
1484         DFG_TYPE_CHECK(
1485             JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr(
1486                 MacroAssembler::Equal,
1487                 structureGPR,
1488                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1489         speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild,
1490             m_jit.branchTest8(
1491                 MacroAssembler::NonZero,
1492                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1493                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1494     }
1495
1496     // It seems that most of the time when programs do a == b where b may be either null/undefined
1497     // or an object, b is usually an object. Balance the branches to make that case fast.
1498     MacroAssembler::Jump rightNotCell =
1499         m_jit.branch32(MacroAssembler::NotEqual, op2TagGPR, TrustedImm32(JSValue::CellTag));
1500
1501     // We know that within this branch, rightChild must be a cell.
1502     if (masqueradesAsUndefinedWatchpointValid) {
1503         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1504         DFG_TYPE_CHECK(
1505             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject,
1506             m_jit.branchPtr(
1507                 MacroAssembler::Equal,
1508                 MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()),
1509                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1510     } else {
1511         m_jit.loadPtr(MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()), structureGPR);
1512         DFG_TYPE_CHECK(
1513             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject,
1514             m_jit.branchPtr(
1515                 MacroAssembler::Equal,
1516                 structureGPR,
1517                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1518         speculationCheck(BadType, JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild,
1519             m_jit.branchTest8(
1520                 MacroAssembler::NonZero,
1521                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1522                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined)));
1523     }
1524
1525     // At this point we know that we can perform a straight-forward equality comparison on pointer
1526     // values because both left and right are pointers to objects that have no special equality
1527     // protocols.
1528     branch32(MacroAssembler::Equal, op1GPR, op2PayloadGPR, taken);
1529
1530     // We know that within this branch, rightChild must not be a cell. Check if that is enough to
1531     // prove that it is either null or undefined.
1532     if (!needsTypeCheck(rightChild, SpecCell | SpecOther))
1533         rightNotCell.link(&m_jit);
1534     else {
1535         jump(notTaken, ForceJump);
1536
1537         rightNotCell.link(&m_jit);
1538         m_jit.move(op2TagGPR, resultGPR);
1539         m_jit.or32(TrustedImm32(1), resultGPR);
1540
1541         typeCheck(
1542             JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, SpecCell | SpecOther,
1543             m_jit.branch32(
1544                 MacroAssembler::NotEqual, resultGPR,
1545                 MacroAssembler::TrustedImm32(JSValue::NullTag)));
1546     }
1547
1548     jump(notTaken);
1549 }
1550
1551 void SpeculativeJIT::compileIntegerCompare(Node* node, MacroAssembler::RelationalCondition condition)
1552 {
1553     SpeculateIntegerOperand op1(this, node->child1());
1554     SpeculateIntegerOperand op2(this, node->child2());
1555     GPRTemporary resultPayload(this);
1556
1557     m_jit.compare32(condition, op1.gpr(), op2.gpr(), resultPayload.gpr());
1558
1559     // If we add a DataFormatBool, we should use it here.
1560     booleanResult(resultPayload.gpr(), node);
1561 }
1562
1563 void SpeculativeJIT::compileDoubleCompare(Node* node, MacroAssembler::DoubleCondition condition)
1564 {
1565     SpeculateDoubleOperand op1(this, node->child1());
1566     SpeculateDoubleOperand op2(this, node->child2());
1567     GPRTemporary resultPayload(this);
1568
1569     m_jit.move(TrustedImm32(1), resultPayload.gpr());
1570     MacroAssembler::Jump trueCase = m_jit.branchDouble(condition, op1.fpr(), op2.fpr());
1571     m_jit.move(TrustedImm32(0), resultPayload.gpr());
1572     trueCase.link(&m_jit);
1573
1574     booleanResult(resultPayload.gpr(), node);
1575 }
1576
1577 void SpeculativeJIT::compileValueAdd(Node* node)
1578 {
1579     JSValueOperand op1(this, node->child1());
1580     JSValueOperand op2(this, node->child2());
1581
1582     GPRReg op1TagGPR = op1.tagGPR();
1583     GPRReg op1PayloadGPR = op1.payloadGPR();
1584     GPRReg op2TagGPR = op2.tagGPR();
1585     GPRReg op2PayloadGPR = op2.payloadGPR();
1586
1587     flushRegisters();
1588
1589     GPRResult2 resultTag(this);
1590     GPRResult resultPayload(this);
1591     if (isKnownNotNumber(node->child1().node()) || isKnownNotNumber(node->child2().node()))
1592         callOperation(operationValueAddNotNumber, resultTag.gpr(), resultPayload.gpr(), op1TagGPR, op1PayloadGPR, op2TagGPR, op2PayloadGPR);
1593     else
1594         callOperation(operationValueAdd, resultTag.gpr(), resultPayload.gpr(), op1TagGPR, op1PayloadGPR, op2TagGPR, op2PayloadGPR);
1595
1596     jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
1597 }
1598
1599 void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse)
1600 {
1601     JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
1602     GPRTemporary resultPayload(this);
1603     GPRReg valueTagGPR = value.tagGPR();
1604     GPRReg valuePayloadGPR = value.payloadGPR();
1605     GPRReg resultPayloadGPR = resultPayload.gpr();
1606     GPRTemporary structure;
1607     GPRReg structureGPR = InvalidGPRReg;
1608
1609     bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid();
1610
1611     if (!masqueradesAsUndefinedWatchpointValid) {
1612         // The masquerades as undefined case will use the structure register, so allocate it here.
1613         // Do this at the top of the function to avoid branching around a register allocation.
1614         GPRTemporary realStructure(this);
1615         structure.adopt(realStructure);
1616         structureGPR = structure.gpr();
1617     }
1618
1619     MacroAssembler::Jump notCell = m_jit.branch32(MacroAssembler::NotEqual, valueTagGPR, TrustedImm32(JSValue::CellTag));
1620     if (masqueradesAsUndefinedWatchpointValid) {
1621         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1622
1623         DFG_TYPE_CHECK(
1624             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject,
1625             m_jit.branchPtr(
1626                 MacroAssembler::Equal,
1627                 MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()),
1628                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1629     } else {
1630         m_jit.loadPtr(MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()), structureGPR);
1631
1632         DFG_TYPE_CHECK(
1633             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject,
1634             m_jit.branchPtr(
1635                 MacroAssembler::Equal,
1636                 structureGPR,
1637                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1638
1639         MacroAssembler::Jump isNotMasqueradesAsUndefined =
1640             m_jit.branchTest8(
1641                 MacroAssembler::Zero,
1642                 MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()),
1643                 MacroAssembler::TrustedImm32(MasqueradesAsUndefined));
1644
1645         speculationCheck(BadType, JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse,
1646             m_jit.branchPtr(
1647                 MacroAssembler::Equal,
1648                 MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()),
1649                 MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin))));
1650
1651         isNotMasqueradesAsUndefined.link(&m_jit);
1652     }
1653     m_jit.move(TrustedImm32(0), resultPayloadGPR);
1654     MacroAssembler::Jump done = m_jit.jump();
1655
1656     notCell.link(&m_jit);
1657
1658     COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag);
1659     if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) {
1660         m_jit.move(valueTagGPR, resultPayloadGPR);
1661         m_jit.or32(TrustedImm32(1), resultPayloadGPR);
1662         typeCheck(
1663             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, SpecCell | SpecOther,
1664             m_jit.branch32(
1665                 MacroAssembler::NotEqual,
1666                 resultPayloadGPR,
1667                 TrustedImm32(JSValue::NullTag)));
1668     }
1669     m_jit.move(TrustedImm32(1), resultPayloadGPR);
1670
1671     done.link(&m_jit);
1672
1673     booleanResult(resultPayloadGPR, m_currentNode);
1674 }
1675
1676 void SpeculativeJIT::compileLogicalNot(Node* node)
1677 {
1678     switch (node->child1().useKind()) {
1679     case BooleanUse: {
1680         SpeculateBooleanOperand value(this, node->child1());
1681         GPRTemporary result(this, value);
1682         m_jit.xor32(TrustedImm32(1), value.gpr(), result.gpr());
1683         booleanResult(result.gpr(), node);
1684         return;
1685     }
1686
1687     case ObjectOrOtherUse: {
1688         compileObjectOrOtherLogicalNot(node->child1());
1689         return;
1690     }
1691
1692     case Int32Use: {
1693         SpeculateIntegerOperand value(this, node->child1());
1694         GPRTemporary resultPayload(this, value);
1695         m_jit.compare32(MacroAssembler::Equal, value.gpr(), MacroAssembler::TrustedImm32(0), resultPayload.gpr());
1696         booleanResult(resultPayload.gpr(), node);
1697         return;
1698     }
1699
1700     case NumberUse: {
1701         SpeculateDoubleOperand value(this, node->child1());
1702         FPRTemporary scratch(this);
1703         GPRTemporary resultPayload(this);
1704         m_jit.move(TrustedImm32(0), resultPayload.gpr());
1705         MacroAssembler::Jump nonZero = m_jit.branchDoubleNonZero(value.fpr(), scratch.fpr());
1706         m_jit.move(TrustedImm32(1), resultPayload.gpr());
1707         nonZero.link(&m_jit);
1708         booleanResult(resultPayload.gpr(), node);
1709         return;
1710     }
1711
1712     case UntypedUse: {
1713         JSValueOperand arg1(this, node->child1());
1714         GPRTemporary resultPayload(this, arg1, false);
1715         GPRReg arg1TagGPR = arg1.tagGPR();
1716         GPRReg arg1PayloadGPR = arg1.payloadGPR();
1717         GPRReg resultPayloadGPR = resultPayload.gpr();
1718
1719         arg1.use();
1720
1721         JITCompiler::Jump slowCase = m_jit.branch32(JITCompiler::NotEqual, arg1TagGPR, TrustedImm32(JSValue::BooleanTag));
1722
1723         m_jit.move(arg1PayloadGPR, resultPayloadGPR);
1724
1725         addSlowPathGenerator(
1726             slowPathCall(
1727                 slowCase, this, dfgConvertJSValueToBoolean, resultPayloadGPR, arg1TagGPR,
1728                 arg1PayloadGPR));
1729
1730         m_jit.xor32(TrustedImm32(1), resultPayloadGPR);
1731         booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly);
1732         return;
1733     }
1734
1735     default:
1736         RELEASE_ASSERT_NOT_REACHED();
1737         break;
1738     }
1739 }
1740
1741 void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BlockIndex taken, BlockIndex notTaken)
1742 {
1743     JSValueOperand value(this, nodeUse, ManualOperandSpeculation);
1744     GPRTemporary scratch(this);
1745     GPRReg valueTagGPR = value.tagGPR();
1746     GPRReg valuePayloadGPR = value.payloadGPR();
1747     GPRReg scratchGPR = scratch.gpr();
1748
1749     MacroAssembler::Jump notCell = m_jit.branch32(MacroAssembler::NotEqual, valueTagGPR, TrustedImm32(JSValue::CellTag));
1750     if (m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
1751         m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
1752
1753         DFG_TYPE_CHECK(
1754             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject,
1755             m_jit.branchPtr(
1756                 MacroAssembler::Equal,
1757                 MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()),
1758                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1759     } else {
1760         m_jit.loadPtr(MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()), scratchGPR);
1761
1762         DFG_TYPE_CHECK(
1763             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject,
1764             m_jit.branchPtr(
1765                 MacroAssembler::Equal,
1766                 scratchGPR,
1767                 MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())));
1768
1769         JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::Zero, MacroAssembler::Address(scratchGPR, Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined));
1770
1771         speculationCheck(BadType, JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse,
1772             m_jit.branchPtr(
1773                 MacroAssembler::Equal,
1774                 MacroAssembler::Address(scratchGPR, Structure::globalObjectOffset()),
1775                 MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin))));
1776
1777         isNotMasqueradesAsUndefined.link(&m_jit);
1778     }
1779     jump(taken, ForceJump);
1780
1781     notCell.link(&m_jit);
1782
1783     COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag);
1784     if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) {
1785         m_jit.move(valueTagGPR, scratchGPR);
1786         m_jit.or32(TrustedImm32(1), scratchGPR);
1787         typeCheck(
1788             JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, SpecCell | SpecOther,
1789             m_jit.branch32(MacroAssembler::NotEqual, scratchGPR, TrustedImm32(JSValue::NullTag)));
1790     }
1791
1792     jump(notTaken);
1793
1794     noResult(m_currentNode);
1795 }
1796
1797 void SpeculativeJIT::emitBranch(Node* node)
1798 {
1799     BlockIndex taken = node->takenBlockIndex();
1800     BlockIndex notTaken = node->notTakenBlockIndex();
1801
1802     switch (node->child1().useKind()) {
1803     case BooleanUse: {
1804         SpeculateBooleanOperand value(this, node->child1());
1805         MacroAssembler::ResultCondition condition = MacroAssembler::NonZero;
1806
1807         if (taken == nextBlock()) {
1808             condition = MacroAssembler::Zero;
1809             BlockIndex tmp = taken;
1810             taken = notTaken;
1811             notTaken = tmp;
1812         }
1813
1814         branchTest32(condition, value.gpr(), TrustedImm32(1), taken);
1815         jump(notTaken);
1816
1817         noResult(node);
1818         return;
1819     }
1820
1821     case ObjectOrOtherUse: {
1822         emitObjectOrOtherBranch(node->child1(), taken, notTaken);
1823         return;
1824     }
1825
1826     case NumberUse:
1827     case Int32Use: {
1828         if (node->child1().useKind() == Int32Use) {
1829             bool invert = false;
1830
1831             if (taken == nextBlock()) {
1832                 invert = true;
1833                 BlockIndex tmp = taken;
1834                 taken = notTaken;
1835                 notTaken = tmp;
1836             }
1837
1838             SpeculateIntegerOperand value(this, node->child1());
1839             branchTest32(invert ? MacroAssembler::Zero : MacroAssembler::NonZero, value.gpr(), taken);
1840         } else {
1841             SpeculateDoubleOperand value(this, node->child1());
1842             FPRTemporary scratch(this);
1843             branchDoubleNonZero(value.fpr(), scratch.fpr(), taken);
1844         }
1845
1846         jump(notTaken);
1847
1848         noResult(node);
1849         return;
1850     }
1851
1852     case UntypedUse: {
1853         JSValueOperand value(this, node->child1());
1854         value.fill();
1855         GPRReg valueTagGPR = value.tagGPR();
1856         GPRReg valuePayloadGPR = value.payloadGPR();
1857
1858         GPRTemporary result(this);
1859         GPRReg resultGPR = result.gpr();
1860
1861         use(node->child1());
1862
1863         JITCompiler::Jump fastPath = m_jit.branch32(JITCompiler::Equal, valueTagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag));
1864         JITCompiler::Jump slowPath = m_jit.branch32(JITCompiler::NotEqual, valueTagGPR, JITCompiler::TrustedImm32(JSValue::BooleanTag));
1865
1866         fastPath.link(&m_jit);
1867         branchTest32(JITCompiler::Zero, valuePayloadGPR, notTaken);
1868         jump(taken, ForceJump);
1869
1870         slowPath.link(&m_jit);
1871         silentSpillAllRegisters(resultGPR);
1872         callOperation(dfgConvertJSValueToBoolean, resultGPR, valueTagGPR, valuePayloadGPR);
1873         silentFillAllRegisters(resultGPR);
1874
1875         branchTest32(JITCompiler::NonZero, resultGPR, taken);
1876         jump(notTaken);
1877
1878         noResult(node, UseChildrenCalledExplicitly);
1879         return;
1880     }
1881
1882     default:
1883         RELEASE_ASSERT_NOT_REACHED();
1884         break;
1885     }
1886 }
1887
1888 template<typename BaseOperandType, typename PropertyOperandType, typename ValueOperandType, typename TagType>
1889 void SpeculativeJIT::compileContiguousPutByVal(Node* node, BaseOperandType& base, PropertyOperandType& property, ValueOperandType& value, GPRReg valuePayloadReg, TagType valueTag)
1890 {
1891     Edge child4 = m_jit.graph().varArgChild(node, 3);
1892
1893     ArrayMode arrayMode = node->arrayMode();
1894
1895     GPRReg baseReg = base.gpr();
1896     GPRReg propertyReg = property.gpr();
1897
1898     StorageOperand storage(this, child4);
1899     GPRReg storageReg = storage.gpr();
1900
1901     if (node->op() == PutByValAlias) {
1902         // Store the value to the array.
1903         GPRReg propertyReg = property.gpr();
1904         m_jit.store32(valueTag, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
1905         m_jit.store32(valuePayloadReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
1906
1907         noResult(node);
1908         return;
1909     }
1910
1911     MacroAssembler::Jump slowCase;
1912
1913     if (arrayMode.isInBounds()) {
1914         speculationCheck(
1915             StoreToHoleOrOutOfBounds, JSValueRegs(), 0,
1916             m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
1917     } else {
1918         MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
1919
1920         slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfVectorLength()));
1921
1922         if (!arrayMode.isOutOfBounds())
1923             speculationCheck(OutOfBounds, JSValueRegs(), 0, slowCase);
1924
1925         m_jit.add32(TrustedImm32(1), propertyReg);
1926         m_jit.store32(propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()));
1927         m_jit.sub32(TrustedImm32(1), propertyReg);
1928
1929         inBounds.link(&m_jit);
1930     }
1931
1932     m_jit.store32(valueTag, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
1933     m_jit.store32(valuePayloadReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
1934
1935     base.use();
1936     property.use();
1937     value.use();
1938     storage.use();
1939
1940     if (arrayMode.isOutOfBounds()) {
1941         addSlowPathGenerator(
1942             slowPathCall(
1943                 slowCase, this,
1944                 m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict,
1945                 NoResult, baseReg, propertyReg, valueTag, valuePayloadReg));
1946     }
1947
1948     noResult(node, UseChildrenCalledExplicitly);
1949 }
1950
1951 void SpeculativeJIT::compile(Node* node)
1952 {
1953     NodeType op = node->op();
1954
1955 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
1956     m_jit.clearRegisterAllocationOffsets();
1957 #endif
1958
1959     switch (op) {
1960     case JSConstant:
1961         initConstantInfo(node);
1962         break;
1963
1964     case PhantomArguments:
1965         initConstantInfo(node);
1966         break;
1967
1968     case WeakJSConstant:
1969         m_jit.addWeakReference(node->weakConstant());
1970         initConstantInfo(node);
1971         break;
1972
1973     case Identity: {
1974         RELEASE_ASSERT_NOT_REACHED();
1975         break;
1976     }
1977
1978     case GetLocal: {
1979         SpeculatedType prediction = node->variableAccessData()->prediction();
1980         AbstractValue& value = m_state.variables().operand(node->local());
1981
1982         // If we have no prediction for this local, then don't attempt to compile.
1983         if (prediction == SpecNone) {
1984             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
1985             break;
1986         }
1987
1988         // If the CFA is tracking this variable and it found that the variable
1989         // cannot have been assigned, then don't attempt to proceed.
1990         if (value.isClear()) {
1991             // FIXME: We should trap instead.
1992             // https://bugs.webkit.org/show_bug.cgi?id=110383
1993             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
1994             break;
1995         }
1996
1997         if (node->variableAccessData()->shouldUseDoubleFormat()) {
1998             FPRTemporary result(this);
1999             m_jit.loadDouble(JITCompiler::addressFor(node->local()), result.fpr());
2000             VirtualRegister virtualRegister = node->virtualRegister();
2001             m_fprs.retain(result.fpr(), virtualRegister, SpillOrderDouble);
2002             m_generationInfo[virtualRegister].initDouble(node, node->refCount(), result.fpr());
2003             break;
2004         }
2005
2006         if (isInt32Speculation(value.m_type)) {
2007             GPRTemporary result(this);
2008             m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
2009
2010             // Like integerResult, but don't useChildren - our children are phi nodes,
2011             // and don't represent values within this dataflow with virtual registers.
2012             VirtualRegister virtualRegister = node->virtualRegister();
2013             m_gprs.retain(result.gpr(), virtualRegister, SpillOrderInteger);
2014             m_generationInfo[virtualRegister].initInteger(node, node->refCount(), result.gpr());
2015             break;
2016         }
2017
2018         if (isCellSpeculation(value.m_type)) {
2019             GPRTemporary result(this);
2020             m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
2021
2022             // Like cellResult, but don't useChildren - our children are phi nodes,
2023             // and don't represent values within this dataflow with virtual registers.
2024             VirtualRegister virtualRegister = node->virtualRegister();
2025             m_gprs.retain(result.gpr(), virtualRegister, SpillOrderCell);
2026             m_generationInfo[virtualRegister].initCell(node, node->refCount(), result.gpr());
2027             break;
2028         }
2029
2030         if (isBooleanSpeculation(value.m_type)) {
2031             GPRTemporary result(this);
2032             m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
2033
2034             // Like booleanResult, but don't useChildren - our children are phi nodes,
2035             // and don't represent values within this dataflow with virtual registers.
2036             VirtualRegister virtualRegister = node->virtualRegister();
2037             m_gprs.retain(result.gpr(), virtualRegister, SpillOrderBoolean);
2038             m_generationInfo[virtualRegister].initBoolean(node, node->refCount(), result.gpr());
2039             break;
2040         }
2041
2042         GPRTemporary result(this);
2043         GPRTemporary tag(this);
2044         m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr());
2045         m_jit.load32(JITCompiler::tagFor(node->local()), tag.gpr());
2046
2047         // Like jsValueResult, but don't useChildren - our children are phi nodes,
2048         // and don't represent values within this dataflow with virtual registers.
2049         VirtualRegister virtualRegister = node->virtualRegister();
2050         m_gprs.retain(result.gpr(), virtualRegister, SpillOrderJS);
2051         m_gprs.retain(tag.gpr(), virtualRegister, SpillOrderJS);
2052
2053         m_generationInfo[virtualRegister].initJSValue(node, node->refCount(), tag.gpr(), result.gpr(), DataFormatJS);
2054         break;
2055     }
2056
2057     case GetLocalUnlinked: {
2058         GPRTemporary payload(this);
2059         GPRTemporary tag(this);
2060         m_jit.load32(JITCompiler::payloadFor(node->unlinkedLocal()), payload.gpr());
2061         m_jit.load32(JITCompiler::tagFor(node->unlinkedLocal()), tag.gpr());
2062         jsValueResult(tag.gpr(), payload.gpr(), node);
2063         break;
2064     }
2065
2066     case MovHintAndCheck: {
2067         compileMovHintAndCheck(node);
2068         break;
2069     }
2070
2071     case InlineStart: {
2072         compileInlineStart(node);
2073         break;
2074     }
2075
2076     case MovHint:
2077     case ZombieHint: {
2078         RELEASE_ASSERT_NOT_REACHED();
2079         break;
2080     }
2081
2082     case SetLocal: {
2083         // SetLocal doubles as a hint as to where a node will be stored and
2084         // as a speculation point. So before we speculate make sure that we
2085         // know where the child of this node needs to go in the virtual
2086         // stack.
2087         compileMovHint(node);
2088
2089         if (node->variableAccessData()->shouldUnboxIfPossible()) {
2090             if (node->variableAccessData()->shouldUseDoubleFormat()) {
2091                 SpeculateDoubleOperand value(this, node->child1());
2092                 m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->local()));
2093                 noResult(node);
2094                 // Indicate that it's no longer necessary to retrieve the value of
2095                 // this bytecode variable from registers or other locations in the stack,
2096                 // but that it is stored as a double.
2097                 recordSetLocal(node->local(), ValueSource(DoubleInJSStack));
2098                 break;
2099             }
2100             SpeculatedType predictedType = node->variableAccessData()->argumentAwarePrediction();
2101             if (m_generationInfo[node->child1()->virtualRegister()].registerFormat() == DataFormatDouble) {
2102                 SpeculateDoubleOperand value(this, node->child1(), ManualOperandSpeculation);
2103                 m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->local()));
2104                 noResult(node);
2105                 recordSetLocal(node->local(), ValueSource(DoubleInJSStack));
2106                 break;
2107             }
2108             if (isInt32Speculation(predictedType)) {
2109                 SpeculateIntegerOperand value(this, node->child1());
2110                 m_jit.store32(value.gpr(), JITCompiler::payloadFor(node->local()));
2111                 noResult(node);
2112                 recordSetLocal(node->local(), ValueSource(Int32InJSStack));
2113                 break;
2114             }
2115             if (isCellSpeculation(predictedType)) {
2116                 SpeculateCellOperand cell(this, node->child1());
2117                 GPRReg cellGPR = cell.gpr();
2118                 m_jit.storePtr(cellGPR, JITCompiler::payloadFor(node->local()));
2119                 noResult(node);
2120                 recordSetLocal(node->local(), ValueSource(CellInJSStack));
2121                 break;
2122             }
2123             if (isBooleanSpeculation(predictedType)) {
2124                 SpeculateBooleanOperand value(this, node->child1());
2125                 m_jit.store32(value.gpr(), JITCompiler::payloadFor(node->local()));
2126                 noResult(node);
2127                 recordSetLocal(node->local(), ValueSource(BooleanInJSStack));
2128                 break;
2129             }
2130         }
2131         JSValueOperand value(this, node->child1());
2132         m_jit.store32(value.payloadGPR(), JITCompiler::payloadFor(node->local()));
2133         m_jit.store32(value.tagGPR(), JITCompiler::tagFor(node->local()));
2134         noResult(node);
2135         recordSetLocal(node->local(), ValueSource(ValueInJSStack));
2136
2137         // If we're storing an arguments object that has been optimized away,
2138         // our variable event stream for OSR exit now reflects the optimized
2139         // value (JSValue()). On the slow path, we want an arguments object
2140         // instead. We add an additional move hint to show OSR exit that it
2141         // needs to reconstruct the arguments object.
2142         if (node->child1()->op() == PhantomArguments)
2143             compileMovHint(node);
2144
2145         break;
2146     }
2147
2148     case SetArgument:
2149         // This is a no-op; it just marks the fact that the argument is being used.
2150         // But it may be profitable to use this as a hook to run speculation checks
2151         // on arguments, thereby allowing us to trivially eliminate such checks if
2152         // the argument is not used.
2153         break;
2154
2155     case BitAnd:
2156     case BitOr:
2157     case BitXor:
2158         if (isInt32Constant(node->child1().node())) {
2159             SpeculateIntegerOperand op2(this, node->child2());
2160             GPRTemporary result(this, op2);
2161
2162             bitOp(op, valueOfInt32Constant(node->child1().node()), op2.gpr(), result.gpr());
2163
2164             integerResult(result.gpr(), node);
2165         } else if (isInt32Constant(node->child2().node())) {
2166             SpeculateIntegerOperand op1(this, node->child1());
2167             GPRTemporary result(this, op1);
2168
2169             bitOp(op, valueOfInt32Constant(node->child2().node()), op1.gpr(), result.gpr());
2170
2171             integerResult(result.gpr(), node);
2172         } else {
2173             SpeculateIntegerOperand op1(this, node->child1());
2174             SpeculateIntegerOperand op2(this, node->child2());
2175             GPRTemporary result(this, op1, op2);
2176
2177             GPRReg reg1 = op1.gpr();
2178             GPRReg reg2 = op2.gpr();
2179             bitOp(op, reg1, reg2, result.gpr());
2180
2181             integerResult(result.gpr(), node);
2182         }
2183         break;
2184
2185     case BitRShift:
2186     case BitLShift:
2187     case BitURShift:
2188         if (isInt32Constant(node->child2().node())) {
2189             SpeculateIntegerOperand op1(this, node->child1());
2190             GPRTemporary result(this, op1);
2191
2192             shiftOp(op, op1.gpr(), valueOfInt32Constant(node->child2().node()) & 0x1f, result.gpr());
2193
2194             integerResult(result.gpr(), node);
2195         } else {
2196             // Do not allow shift amount to be used as the result, MacroAssembler does not permit this.
2197             SpeculateIntegerOperand op1(this, node->child1());
2198             SpeculateIntegerOperand op2(this, node->child2());
2199             GPRTemporary result(this, op1);
2200
2201             GPRReg reg1 = op1.gpr();
2202             GPRReg reg2 = op2.gpr();
2203             shiftOp(op, reg1, reg2, result.gpr());
2204
2205             integerResult(result.gpr(), node);
2206         }
2207         break;
2208
2209     case UInt32ToNumber: {
2210         compileUInt32ToNumber(node);
2211         break;
2212     }
2213
2214     case DoubleAsInt32: {
2215         compileDoubleAsInt32(node);
2216         break;
2217     }
2218
2219     case ValueToInt32: {
2220         compileValueToInt32(node);
2221         break;
2222     }
2223
2224     case Int32ToDouble:
2225     case ForwardInt32ToDouble: {
2226         compileInt32ToDouble(node);
2227         break;
2228     }
2229
2230     case ValueAdd:
2231     case ArithAdd:
2232         compileAdd(node);
2233         break;
2234
2235     case MakeRope:
2236         compileMakeRope(node);
2237         break;
2238
2239     case ArithSub:
2240         compileArithSub(node);
2241         break;
2242
2243     case ArithNegate:
2244         compileArithNegate(node);
2245         break;
2246
2247     case ArithMul:
2248         compileArithMul(node);
2249         break;
2250
2251     case ArithIMul:
2252         compileArithIMul(node);
2253         break;
2254
2255     case ArithDiv: {
2256         switch (node->binaryUseKind()) {
2257         case Int32Use: {
2258 #if CPU(X86)
2259             compileIntegerArithDivForX86(node);
2260 #elif CPU(ARM64)
2261             compileIntegerArithDivForARM64(node);
2262 #elif CPU(APPLE_ARMV7S)
2263             compileIntegerArithDivForARMv7s(node);
2264 #else // CPU type without integer divide
2265             RELEASE_ASSERT_NOT_REACHED(); // should have been coverted into a double divide.
2266 #endif
2267             break;
2268         }
2269
2270         case NumberUse: {
2271             SpeculateDoubleOperand op1(this, node->child1());
2272             SpeculateDoubleOperand op2(this, node->child2());
2273             FPRTemporary result(this, op1);
2274
2275             FPRReg reg1 = op1.fpr();
2276             FPRReg reg2 = op2.fpr();
2277             m_jit.divDouble(reg1, reg2, result.fpr());
2278
2279             doubleResult(result.fpr(), node);
2280             break;
2281         }
2282
2283         default:
2284             RELEASE_ASSERT_NOT_REACHED();
2285             break;
2286         }
2287         break;
2288     }
2289
2290     case ArithMod: {
2291         compileArithMod(node);
2292         break;
2293     }
2294
2295     case ArithAbs: {
2296         switch (node->child1().useKind()) {
2297         case Int32Use: {
2298             SpeculateIntegerOperand op1(this, node->child1());
2299             GPRTemporary result(this, op1);
2300             GPRTemporary scratch(this);
2301
2302             m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr());
2303             m_jit.rshift32(result.gpr(), MacroAssembler::TrustedImm32(31), scratch.gpr());
2304             m_jit.add32(scratch.gpr(), result.gpr());
2305             m_jit.xor32(scratch.gpr(), result.gpr());
2306             speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, result.gpr(), MacroAssembler::TrustedImm32(1 << 31)));
2307             integerResult(result.gpr(), node);
2308             break;
2309         }
2310
2311
2312         case NumberUse: {
2313             SpeculateDoubleOperand op1(this, node->child1());
2314             FPRTemporary result(this);
2315
2316             m_jit.absDouble(op1.fpr(), result.fpr());
2317             doubleResult(result.fpr(), node);
2318             break;
2319         }
2320
2321         default:
2322             RELEASE_ASSERT_NOT_REACHED();
2323             break;
2324         }
2325         break;
2326     }
2327
2328     case ArithMin:
2329     case ArithMax: {
2330         switch (node->binaryUseKind()) {
2331         case Int32Use: {
2332             SpeculateStrictInt32Operand op1(this, node->child1());
2333             SpeculateStrictInt32Operand op2(this, node->child2());
2334             GPRTemporary result(this, op1);
2335
2336             GPRReg op1GPR = op1.gpr();
2337             GPRReg op2GPR = op2.gpr();
2338             GPRReg resultGPR = result.gpr();
2339
2340             MacroAssembler::Jump op1Less = m_jit.branch32(op == ArithMin ? MacroAssembler::LessThan : MacroAssembler::GreaterThan, op1GPR, op2GPR);
2341             m_jit.move(op2GPR, resultGPR);
2342             if (op1GPR != resultGPR) {
2343                 MacroAssembler::Jump done = m_jit.jump();
2344                 op1Less.link(&m_jit);
2345                 m_jit.move(op1GPR, resultGPR);
2346                 done.link(&m_jit);
2347             } else
2348                 op1Less.link(&m_jit);
2349
2350             integerResult(resultGPR, node);
2351             break;
2352         }
2353
2354         case NumberUse: {
2355             SpeculateDoubleOperand op1(this, node->child1());
2356             SpeculateDoubleOperand op2(this, node->child2());
2357             FPRTemporary result(this, op1);
2358
2359             FPRReg op1FPR = op1.fpr();
2360             FPRReg op2FPR = op2.fpr();
2361             FPRReg resultFPR = result.fpr();
2362
2363             MacroAssembler::JumpList done;
2364
2365             MacroAssembler::Jump op1Less = m_jit.branchDouble(op == ArithMin ? MacroAssembler::DoubleLessThan : MacroAssembler::DoubleGreaterThan, op1FPR, op2FPR);
2366
2367             // op2 is eather the lesser one or one of then is NaN
2368             MacroAssembler::Jump op2Less = m_jit.branchDouble(op == ArithMin ? MacroAssembler::DoubleGreaterThanOrEqual : MacroAssembler::DoubleLessThanOrEqual, op1FPR, op2FPR);
2369
2370             // Unordered case. We don't know which of op1, op2 is NaN. Manufacture NaN by adding
2371             // op1 + op2 and putting it into result.
2372             m_jit.addDouble(op1FPR, op2FPR, resultFPR);
2373             done.append(m_jit.jump());
2374
2375             op2Less.link(&m_jit);
2376             m_jit.moveDouble(op2FPR, resultFPR);
2377
2378             if (op1FPR != resultFPR) {
2379                 done.append(m_jit.jump());
2380
2381                 op1Less.link(&m_jit);
2382                 m_jit.moveDouble(op1FPR, resultFPR);
2383             } else
2384                 op1Less.link(&m_jit);
2385
2386             done.link(&m_jit);
2387
2388             doubleResult(resultFPR, node);
2389             break;
2390         }
2391
2392         default:
2393             RELEASE_ASSERT_NOT_REACHED();
2394             break;
2395         }
2396         break;
2397     }
2398
2399     case ArithSqrt: {
2400         SpeculateDoubleOperand op1(this, node->child1());
2401         FPRTemporary result(this, op1);
2402
2403         m_jit.sqrtDouble(op1.fpr(), result.fpr());
2404
2405         doubleResult(result.fpr(), node);
2406         break;
2407     }
2408
2409     case LogicalNot:
2410         compileLogicalNot(node);
2411         break;
2412
2413     case CompareLess:
2414         if (compare(node, JITCompiler::LessThan, JITCompiler::DoubleLessThan, operationCompareLess))
2415             return;
2416         break;
2417
2418     case CompareLessEq:
2419         if (compare(node, JITCompiler::LessThanOrEqual, JITCompiler::DoubleLessThanOrEqual, operationCompareLessEq))
2420             return;
2421         break;
2422
2423     case CompareGreater:
2424         if (compare(node, JITCompiler::GreaterThan, JITCompiler::DoubleGreaterThan, operationCompareGreater))
2425             return;
2426         break;
2427
2428     case CompareGreaterEq:
2429         if (compare(node, JITCompiler::GreaterThanOrEqual, JITCompiler::DoubleGreaterThanOrEqual, operationCompareGreaterEq))
2430             return;
2431         break;
2432
2433     case CompareEqConstant:
2434         ASSERT(isNullConstant(node->child2().node()));
2435         if (nonSpeculativeCompareNull(node, node->child1()))
2436             return;
2437         break;
2438
2439     case CompareEq:
2440         if (compare(node, JITCompiler::Equal, JITCompiler::DoubleEqual, operationCompareEq))
2441             return;
2442         break;
2443
2444     case CompareStrictEqConstant:
2445         if (compileStrictEqForConstant(node, node->child1(), valueOfJSConstant(node->child2().node())))
2446             return;
2447         break;
2448
2449     case CompareStrictEq:
2450         if (compileStrictEq(node))
2451             return;
2452         break;
2453
2454     case StringCharCodeAt: {
2455         compileGetCharCodeAt(node);
2456         break;
2457     }
2458
2459     case StringCharAt: {
2460         // Relies on StringCharAt node having same basic layout as GetByVal
2461         compileGetByValOnString(node);
2462         break;
2463     }
2464
2465     case StringFromCharCode: {
2466         compileFromCharCode(node);
2467         break;
2468     }
2469
2470     case CheckArray: {
2471         checkArray(node);
2472         break;
2473     }
2474
2475     case Arrayify:
2476     case ArrayifyToStructure: {
2477         arrayify(node);
2478         break;
2479     }
2480
2481     case GetByVal: {
2482         switch (node->arrayMode().type()) {
2483         case Array::SelectUsingPredictions:
2484         case Array::ForceExit:
2485             RELEASE_ASSERT_NOT_REACHED();
2486             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
2487             break;
2488         case Array::Generic: {
2489             SpeculateCellOperand base(this, node->child1()); // Save a register, speculate cell. We'll probably be right.
2490             JSValueOperand property(this, node->child2());
2491             GPRReg baseGPR = base.gpr();
2492             GPRReg propertyTagGPR = property.tagGPR();
2493             GPRReg propertyPayloadGPR = property.payloadGPR();
2494
2495             flushRegisters();
2496             GPRResult2 resultTag(this);
2497             GPRResult resultPayload(this);
2498             callOperation(operationGetByValCell, resultTag.gpr(), resultPayload.gpr(), baseGPR, propertyTagGPR, propertyPayloadGPR);
2499
2500             jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
2501             break;
2502         }
2503         case Array::Int32:
2504         case Array::Contiguous: {
2505             if (node->arrayMode().isInBounds()) {
2506                 SpeculateStrictInt32Operand property(this, node->child2());
2507                 StorageOperand storage(this, node->child3());
2508
2509                 GPRReg propertyReg = property.gpr();
2510                 GPRReg storageReg = storage.gpr();
2511
2512                 if (!m_compileOkay)
2513                     return;
2514
2515                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2516
2517                 GPRTemporary resultPayload(this);
2518                 if (node->arrayMode().type() == Array::Int32) {
2519                     speculationCheck(
2520                         OutOfBounds, JSValueRegs(), 0,
2521                         m_jit.branch32(
2522                             MacroAssembler::Equal,
2523                             MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)),
2524                             TrustedImm32(JSValue::EmptyValueTag)));
2525                     m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayload.gpr());
2526                     integerResult(resultPayload.gpr(), node);
2527                     break;
2528                 }
2529
2530                 GPRTemporary resultTag(this);
2531                 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTag.gpr());
2532                 speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, resultTag.gpr(), TrustedImm32(JSValue::EmptyValueTag)));
2533                 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayload.gpr());
2534                 jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
2535                 break;
2536             }
2537
2538             SpeculateCellOperand base(this, node->child1());
2539             SpeculateStrictInt32Operand property(this, node->child2());
2540             StorageOperand storage(this, node->child3());
2541
2542             GPRReg baseReg = base.gpr();
2543             GPRReg propertyReg = property.gpr();
2544             GPRReg storageReg = storage.gpr();
2545
2546             if (!m_compileOkay)
2547                 return;
2548
2549             GPRTemporary resultTag(this);
2550             GPRTemporary resultPayload(this);
2551             GPRReg resultTagReg = resultTag.gpr();
2552             GPRReg resultPayloadReg = resultPayload.gpr();
2553
2554             MacroAssembler::JumpList slowCases;
2555
2556             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2557
2558             m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTagReg);
2559             m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayloadReg);
2560             slowCases.append(m_jit.branch32(MacroAssembler::Equal, resultTagReg, TrustedImm32(JSValue::EmptyValueTag)));
2561
2562             addSlowPathGenerator(
2563                 slowPathCall(
2564                     slowCases, this, operationGetByValArrayInt,
2565                     JSValueRegs(resultTagReg, resultPayloadReg), baseReg, propertyReg));
2566
2567             jsValueResult(resultTagReg, resultPayloadReg, node);
2568             break;
2569         }
2570         case Array::Double: {
2571             if (node->arrayMode().isInBounds()) {
2572                 if (node->arrayMode().isSaneChain()) {
2573                     JSGlobalObject* globalObject = m_jit.globalObjectFor(node->codeOrigin);
2574                     ASSERT(globalObject->arrayPrototypeChainIsSane());
2575                     globalObject->arrayPrototype()->structure()->addTransitionWatchpoint(speculationWatchpoint());
2576                     globalObject->objectPrototype()->structure()->addTransitionWatchpoint(speculationWatchpoint());
2577                 }
2578
2579                 SpeculateStrictInt32Operand property(this, node->child2());
2580                 StorageOperand storage(this, node->child3());
2581
2582                 GPRReg propertyReg = property.gpr();
2583                 GPRReg storageReg = storage.gpr();
2584
2585                 if (!m_compileOkay)
2586                     return;
2587
2588                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2589
2590                 FPRTemporary result(this);
2591                 m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), result.fpr());
2592                 if (!node->arrayMode().isSaneChain())
2593                     speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, result.fpr(), result.fpr()));
2594                 doubleResult(result.fpr(), node);
2595                 break;
2596             }
2597
2598             SpeculateCellOperand base(this, node->child1());
2599             SpeculateStrictInt32Operand property(this, node->child2());
2600             StorageOperand storage(this, node->child3());
2601
2602             GPRReg baseReg = base.gpr();
2603             GPRReg propertyReg = property.gpr();
2604             GPRReg storageReg = storage.gpr();
2605
2606             if (!m_compileOkay)
2607                 return;
2608
2609             GPRTemporary resultTag(this);
2610             GPRTemporary resultPayload(this);
2611             FPRTemporary temp(this);
2612             GPRReg resultTagReg = resultTag.gpr();
2613             GPRReg resultPayloadReg = resultPayload.gpr();
2614             FPRReg tempReg = temp.fpr();
2615
2616             MacroAssembler::JumpList slowCases;
2617
2618             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())));
2619
2620             m_jit.loadDouble(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), tempReg);
2621             slowCases.append(m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempReg, tempReg));
2622             boxDouble(tempReg, resultTagReg, resultPayloadReg);
2623
2624             addSlowPathGenerator(
2625                 slowPathCall(
2626                     slowCases, this, operationGetByValArrayInt,
2627                     JSValueRegs(resultTagReg, resultPayloadReg), baseReg, propertyReg));
2628
2629             jsValueResult(resultTagReg, resultPayloadReg, node);
2630             break;
2631         }
2632         case Array::ArrayStorage:
2633         case Array::SlowPutArrayStorage: {
2634             if (node->arrayMode().isInBounds()) {
2635                 SpeculateStrictInt32Operand property(this, node->child2());
2636                 StorageOperand storage(this, node->child3());
2637                 GPRReg propertyReg = property.gpr();
2638                 GPRReg storageReg = storage.gpr();
2639
2640                 if (!m_compileOkay)
2641                     return;
2642
2643                 speculationCheck(OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset())));
2644
2645                 GPRTemporary resultTag(this);
2646                 GPRTemporary resultPayload(this);
2647
2648                 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTag.gpr());
2649                 speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, resultTag.gpr(), TrustedImm32(JSValue::EmptyValueTag)));
2650                 m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayload.gpr());
2651
2652                 jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
2653                 break;
2654             }
2655
2656             SpeculateCellOperand base(this, node->child1());
2657             SpeculateStrictInt32Operand property(this, node->child2());
2658             StorageOperand storage(this, node->child3());
2659             GPRReg propertyReg = property.gpr();
2660             GPRReg storageReg = storage.gpr();
2661             GPRReg baseReg = base.gpr();
2662
2663             if (!m_compileOkay)
2664                 return;
2665
2666             GPRTemporary resultTag(this);
2667             GPRTemporary resultPayload(this);
2668             GPRReg resultTagReg = resultTag.gpr();
2669             GPRReg resultPayloadReg = resultPayload.gpr();
2670
2671             JITCompiler::Jump outOfBounds = m_jit.branch32(
2672                 MacroAssembler::AboveOrEqual, propertyReg,
2673                 MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset()));
2674
2675             m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTagReg);
2676             JITCompiler::Jump hole = m_jit.branch32(
2677                 MacroAssembler::Equal, resultTag.gpr(), TrustedImm32(JSValue::EmptyValueTag));
2678             m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayloadReg);
2679
2680             JITCompiler::JumpList slowCases;
2681             slowCases.append(outOfBounds);
2682             slowCases.append(hole);
2683             addSlowPathGenerator(
2684                 slowPathCall(
2685                     slowCases, this, operationGetByValArrayInt,
2686                     JSValueRegs(resultTagReg, resultPayloadReg),
2687                     baseReg, propertyReg));
2688
2689             jsValueResult(resultTagReg, resultPayloadReg, node);
2690             break;
2691         }
2692         case Array::String:
2693             compileGetByValOnString(node);
2694             break;
2695         case Array::Arguments:
2696             compileGetByValOnArguments(node);
2697             break;
2698         case Array::Int8Array:
2699             compileGetByValOnIntTypedArray(m_jit.vm()->int8ArrayDescriptor(), node, sizeof(int8_t), SignedTypedArray);
2700             break;
2701         case Array::Int16Array:
2702             compileGetByValOnIntTypedArray(m_jit.vm()->int16ArrayDescriptor(), node, sizeof(int16_t), SignedTypedArray);
2703             break;
2704         case Array::Int32Array:
2705             compileGetByValOnIntTypedArray(m_jit.vm()->int32ArrayDescriptor(), node, sizeof(int32_t), SignedTypedArray);
2706             break;
2707         case Array::Uint8Array:
2708             compileGetByValOnIntTypedArray(m_jit.vm()->uint8ArrayDescriptor(), node, sizeof(uint8_t), UnsignedTypedArray);
2709             break;
2710         case Array::Uint8ClampedArray:
2711             compileGetByValOnIntTypedArray(m_jit.vm()->uint8ClampedArrayDescriptor(), node, sizeof(uint8_t), UnsignedTypedArray);
2712             break;
2713         case Array::Uint16Array:
2714             compileGetByValOnIntTypedArray(m_jit.vm()->uint16ArrayDescriptor(), node, sizeof(uint16_t), UnsignedTypedArray);
2715             break;
2716         case Array::Uint32Array:
2717             compileGetByValOnIntTypedArray(m_jit.vm()->uint32ArrayDescriptor(), node, sizeof(uint32_t), UnsignedTypedArray);
2718             break;
2719         case Array::Float32Array:
2720             compileGetByValOnFloatTypedArray(m_jit.vm()->float32ArrayDescriptor(), node, sizeof(float));
2721             break;
2722         case Array::Float64Array:
2723             compileGetByValOnFloatTypedArray(m_jit.vm()->float64ArrayDescriptor(), node, sizeof(double));
2724             break;
2725         default:
2726             RELEASE_ASSERT_NOT_REACHED();
2727             break;
2728         }
2729         break;
2730     }
2731
2732     case PutByVal:
2733     case PutByValAlias: {
2734         Edge child1 = m_jit.graph().varArgChild(node, 0);
2735         Edge child2 = m_jit.graph().varArgChild(node, 1);
2736         Edge child3 = m_jit.graph().varArgChild(node, 2);
2737         Edge child4 = m_jit.graph().varArgChild(node, 3);
2738
2739         ArrayMode arrayMode = node->arrayMode().modeForPut();
2740         bool alreadyHandled = false;
2741
2742         switch (arrayMode.type()) {
2743         case Array::SelectUsingPredictions:
2744         case Array::ForceExit:
2745             RELEASE_ASSERT_NOT_REACHED();
2746             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
2747             alreadyHandled = true;
2748             break;
2749         case Array::Generic: {
2750             ASSERT(node->op() == PutByVal);
2751
2752             SpeculateCellOperand base(this, child1); // Save a register, speculate cell. We'll probably be right.
2753             JSValueOperand property(this, child2);
2754             JSValueOperand value(this, child3);
2755             GPRReg baseGPR = base.gpr();
2756             GPRReg propertyTagGPR = property.tagGPR();
2757             GPRReg propertyPayloadGPR = property.payloadGPR();
2758             GPRReg valueTagGPR = value.tagGPR();
2759             GPRReg valuePayloadGPR = value.payloadGPR();
2760
2761             flushRegisters();
2762             callOperation(m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict, baseGPR, propertyTagGPR, propertyPayloadGPR, valueTagGPR, valuePayloadGPR);
2763
2764             noResult(node);
2765             alreadyHandled = true;
2766             break;
2767         }
2768         default:
2769             break;
2770         }
2771
2772         if (alreadyHandled)
2773             break;
2774
2775         SpeculateCellOperand base(this, child1);
2776         SpeculateStrictInt32Operand property(this, child2);
2777
2778         GPRReg baseReg = base.gpr();
2779         GPRReg propertyReg = property.gpr();
2780
2781         switch (arrayMode.type()) {
2782         case Array::Int32: {
2783             SpeculateIntegerOperand value(this, child3);
2784
2785             GPRReg valuePayloadReg = value.gpr();
2786
2787             if (!m_compileOkay)
2788                 return;
2789
2790             compileContiguousPutByVal(node, base, property, value, valuePayloadReg, TrustedImm32(JSValue::Int32Tag));
2791             break;
2792         }
2793         case Array::Contiguous: {
2794             JSValueOperand value(this, child3);
2795
2796             GPRReg valueTagReg = value.tagGPR();
2797             GPRReg valuePayloadReg = value.payloadGPR();
2798
2799             if (!m_compileOkay)
2800                 return;
2801
2802             if (Heap::isWriteBarrierEnabled()) {
2803                 GPRTemporary scratch(this);
2804                 writeBarrier(baseReg, valueTagReg, child3, WriteBarrierForPropertyAccess, scratch.gpr());
2805             }
2806
2807             compileContiguousPutByVal(node, base, property, value, valuePayloadReg, valueTagReg);
2808             break;
2809         }
2810         case Array::Double: {
2811             compileDoublePutByVal(node, base, property);
2812             break;
2813         }
2814         case Array::ArrayStorage:
2815         case Array::SlowPutArrayStorage: {
2816             JSValueOperand value(this, child3);
2817
2818             GPRReg valueTagReg = value.tagGPR();
2819             GPRReg valuePayloadReg = value.payloadGPR();
2820
2821             if (!m_compileOkay)
2822                 return;
2823
2824             {
2825                 GPRTemporary scratch(this);
2826                 GPRReg scratchReg = scratch.gpr();
2827                 writeBarrier(baseReg, valueTagReg, child3, WriteBarrierForPropertyAccess, scratchReg);
2828             }
2829
2830             StorageOperand storage(this, child4);
2831             GPRReg storageReg = storage.gpr();
2832
2833             if (node->op() == PutByValAlias) {
2834                 // Store the value to the array.
2835                 GPRReg propertyReg = property.gpr();
2836                 m_jit.store32(value.tagGPR(), MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
2837                 m_jit.store32(value.payloadGPR(), MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
2838
2839                 noResult(node);
2840                 break;
2841             }
2842
2843             MacroAssembler::JumpList slowCases;
2844
2845             MacroAssembler::Jump beyondArrayBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::vectorLengthOffset()));
2846             if (!arrayMode.isOutOfBounds())
2847                 speculationCheck(OutOfBounds, JSValueRegs(), 0, beyondArrayBounds);
2848             else
2849                 slowCases.append(beyondArrayBounds);
2850
2851             // Check if we're writing to a hole; if so increment m_numValuesInVector.
2852             if (arrayMode.isInBounds()) {
2853                 speculationCheck(
2854                     StoreToHole, JSValueRegs(), 0,
2855                     m_jit.branch32(MacroAssembler::Equal, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), TrustedImm32(JSValue::EmptyValueTag)));
2856             } else {
2857                 MacroAssembler::Jump notHoleValue = m_jit.branch32(MacroAssembler::NotEqual, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), TrustedImm32(JSValue::EmptyValueTag));
2858                 if (arrayMode.isSlowPut()) {
2859                     // This is sort of strange. If we wanted to optimize this code path, we would invert
2860                     // the above branch. But it's simply not worth it since this only happens if we're
2861                     // already having a bad time.
2862                     slowCases.append(m_jit.jump());
2863                 } else {
2864                     m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageReg, ArrayStorage::numValuesInVectorOffset()));
2865
2866                     // If we're writing to a hole we might be growing the array;
2867                     MacroAssembler::Jump lengthDoesNotNeedUpdate = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::lengthOffset()));
2868                     m_jit.add32(TrustedImm32(1), propertyReg);
2869                     m_jit.store32(propertyReg, MacroAssembler::Address(storageReg, ArrayStorage::lengthOffset()));
2870                     m_jit.sub32(TrustedImm32(1), propertyReg);
2871
2872                     lengthDoesNotNeedUpdate.link(&m_jit);
2873                 }
2874                 notHoleValue.link(&m_jit);
2875             }
2876
2877             // Store the value to the array.
2878             m_jit.store32(valueTagReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
2879             m_jit.store32(valuePayloadReg, MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
2880
2881             base.use();
2882             property.use();
2883             value.use();
2884             storage.use();
2885
2886             if (!slowCases.empty()) {
2887                 addSlowPathGenerator(
2888                     slowPathCall(
2889                         slowCases, this,
2890                         m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict,
2891                         NoResult, baseReg, propertyReg, valueTagReg, valuePayloadReg));
2892             }
2893
2894             noResult(node, UseChildrenCalledExplicitly);
2895             break;
2896         }
2897
2898         case Array::Arguments:
2899             // FIXME: we could at some point make this work. Right now we're assuming that the register
2900             // pressure would be too great.
2901             RELEASE_ASSERT_NOT_REACHED();
2902             break;
2903
2904         case Array::Int8Array:
2905             compilePutByValForIntTypedArray(m_jit.vm()->int8ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int8_t), SignedTypedArray);
2906             break;
2907
2908         case Array::Int16Array:
2909             compilePutByValForIntTypedArray(m_jit.vm()->int16ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int16_t), SignedTypedArray);
2910             break;
2911
2912         case Array::Int32Array:
2913             compilePutByValForIntTypedArray(m_jit.vm()->int32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int32_t), SignedTypedArray);
2914             break;
2915
2916         case Array::Uint8Array:
2917             compilePutByValForIntTypedArray(m_jit.vm()->uint8ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint8_t), UnsignedTypedArray);
2918             break;
2919
2920         case Array::Uint8ClampedArray:
2921             compilePutByValForIntTypedArray(m_jit.vm()->uint8ClampedArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint8_t), UnsignedTypedArray, ClampRounding);
2922             break;
2923
2924         case Array::Uint16Array:
2925             compilePutByValForIntTypedArray(m_jit.vm()->uint16ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint16_t), UnsignedTypedArray);
2926             break;
2927
2928         case Array::Uint32Array:
2929             compilePutByValForIntTypedArray(m_jit.vm()->uint32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint32_t), UnsignedTypedArray);
2930             break;
2931
2932         case Array::Float32Array:
2933             compilePutByValForFloatTypedArray(m_jit.vm()->float32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(float));
2934             break;
2935
2936         case Array::Float64Array:
2937             compilePutByValForFloatTypedArray(m_jit.vm()->float64ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(double));
2938             break;
2939
2940         default:
2941             RELEASE_ASSERT_NOT_REACHED();
2942             break;
2943         }
2944         break;
2945     }
2946
2947     case RegExpExec: {
2948         if (compileRegExpExec(node))
2949             return;
2950
2951         if (!node->adjustedRefCount()) {
2952             SpeculateCellOperand base(this, node->child1());
2953             SpeculateCellOperand argument(this, node->child2());
2954             GPRReg baseGPR = base.gpr();
2955             GPRReg argumentGPR = argument.gpr();
2956
2957             flushRegisters();
2958             GPRResult result(this);
2959             callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR);
2960
2961             // Must use jsValueResult because otherwise we screw up register
2962             // allocation, which thinks that this node has a result.
2963             booleanResult(result.gpr(), node);
2964             break;
2965         }
2966
2967         SpeculateCellOperand base(this, node->child1());
2968         SpeculateCellOperand argument(this, node->child2());
2969         GPRReg baseGPR = base.gpr();
2970         GPRReg argumentGPR = argument.gpr();
2971
2972         flushRegisters();
2973         GPRResult2 resultTag(this);
2974         GPRResult resultPayload(this);
2975         callOperation(operationRegExpExec, resultTag.gpr(), resultPayload.gpr(), baseGPR, argumentGPR);
2976
2977         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
2978         break;
2979     }
2980
2981     case RegExpTest: {
2982         SpeculateCellOperand base(this, node->child1());
2983         SpeculateCellOperand argument(this, node->child2());
2984         GPRReg baseGPR = base.gpr();
2985         GPRReg argumentGPR = argument.gpr();
2986
2987         flushRegisters();
2988         GPRResult result(this);
2989         callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR);
2990
2991         // If we add a DataFormatBool, we should use it here.
2992         booleanResult(result.gpr(), node);
2993         break;
2994     }
2995
2996     case ArrayPush: {
2997         ASSERT(node->arrayMode().isJSArray());
2998
2999         SpeculateCellOperand base(this, node->child1());
3000         GPRTemporary storageLength(this);
3001
3002         GPRReg baseGPR = base.gpr();
3003         GPRReg storageLengthGPR = storageLength.gpr();
3004
3005         StorageOperand storage(this, node->child3());
3006         GPRReg storageGPR = storage.gpr();
3007
3008         switch (node->arrayMode().type()) {
3009         case Array::Int32: {
3010             SpeculateIntegerOperand value(this, node->child2());
3011             GPRReg valuePayloadGPR = value.gpr();
3012
3013             m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
3014             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3015             m_jit.store32(TrustedImm32(JSValue::Int32Tag), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3016             m_jit.store32(valuePayloadGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3017             m_jit.add32(TrustedImm32(1), storageLengthGPR);
3018             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3019             m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR);
3020
3021             addSlowPathGenerator(
3022                 slowPathCall(
3023                     slowPath, this, operationArrayPush,
3024                     JSValueRegs(storageGPR, storageLengthGPR),
3025                     TrustedImm32(JSValue::Int32Tag), valuePayloadGPR, baseGPR));
3026
3027             jsValueResult(storageGPR, storageLengthGPR, node);
3028             break;
3029         }
3030
3031         case Array::Contiguous: {
3032             JSValueOperand value(this, node->child2());
3033             GPRReg valueTagGPR = value.tagGPR();
3034             GPRReg valuePayloadGPR = value.payloadGPR();
3035
3036             if (Heap::isWriteBarrierEnabled()) {
3037                 GPRTemporary scratch(this);
3038                 writeBarrier(baseGPR, valueTagGPR, node->child2(), WriteBarrierForPropertyAccess, scratch.gpr(), storageLengthGPR);
3039             }
3040
3041             m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
3042             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3043             m_jit.store32(valueTagGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3044             m_jit.store32(valuePayloadGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3045             m_jit.add32(TrustedImm32(1), storageLengthGPR);
3046             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3047             m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR);
3048
3049             addSlowPathGenerator(
3050                 slowPathCall(
3051                     slowPath, this, operationArrayPush,
3052                     JSValueRegs(storageGPR, storageLengthGPR),
3053                     valueTagGPR, valuePayloadGPR, baseGPR));
3054
3055             jsValueResult(storageGPR, storageLengthGPR, node);
3056             break;
3057         }
3058
3059         case Array::Double: {
3060             SpeculateDoubleOperand value(this, node->child2());
3061             FPRReg valueFPR = value.fpr();
3062
3063             DFG_TYPE_CHECK(
3064                 JSValueRegs(), node->child2(), SpecRealNumber,
3065                 m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, valueFPR, valueFPR));
3066
3067             m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR);
3068             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3069             m_jit.storeDouble(valueFPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight));
3070             m_jit.add32(TrustedImm32(1), storageLengthGPR);
3071             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3072             m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR);
3073
3074             addSlowPathGenerator(
3075                 slowPathCall(
3076                     slowPath, this, operationArrayPushDouble,
3077                     JSValueRegs(storageGPR, storageLengthGPR),
3078                     valueFPR, baseGPR));
3079
3080             jsValueResult(storageGPR, storageLengthGPR, node);
3081             break;
3082         }
3083
3084         case Array::ArrayStorage: {
3085             JSValueOperand value(this, node->child2());
3086             GPRReg valueTagGPR = value.tagGPR();
3087             GPRReg valuePayloadGPR = value.payloadGPR();
3088
3089             if (Heap::isWriteBarrierEnabled()) {
3090                 GPRTemporary scratch(this);
3091                 writeBarrier(baseGPR, valueTagGPR, node->child2(), WriteBarrierForPropertyAccess, scratch.gpr(), storageLengthGPR);
3092             }
3093
3094             m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
3095
3096             // Refuse to handle bizarre lengths.
3097             speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Above, storageLengthGPR, TrustedImm32(0x7ffffffe)));
3098
3099             MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
3100
3101             m_jit.store32(valueTagGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3102             m_jit.store32(valuePayloadGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3103
3104             m_jit.add32(TrustedImm32(1), storageLengthGPR);
3105             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
3106             m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
3107             m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR);
3108
3109             addSlowPathGenerator(slowPathCall(slowPath, this, operationArrayPush, JSValueRegs(storageGPR, storageLengthGPR), valueTagGPR, valuePayloadGPR, baseGPR));
3110
3111             jsValueResult(storageGPR, storageLengthGPR, node);
3112             break;
3113         }
3114
3115         default:
3116             CRASH();
3117             break;
3118         }
3119         break;
3120     }
3121
3122     case ArrayPop: {
3123         ASSERT(node->arrayMode().isJSArray());
3124
3125         SpeculateCellOperand base(this, node->child1());
3126         StorageOperand storage(this, node->child2());
3127         GPRTemporary valueTag(this);
3128         GPRTemporary valuePayload(this);
3129
3130         GPRReg baseGPR = base.gpr();
3131         GPRReg valueTagGPR = valueTag.gpr();
3132         GPRReg valuePayloadGPR = valuePayload.gpr();
3133         GPRReg storageGPR = storage.gpr();
3134
3135         switch (node->arrayMode().type()) {
3136         case Array::Int32:
3137         case Array::Contiguous: {
3138             m_jit.load32(
3139                 MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), valuePayloadGPR);
3140             MacroAssembler::Jump undefinedCase =
3141                 m_jit.branchTest32(MacroAssembler::Zero, valuePayloadGPR);
3142             m_jit.sub32(TrustedImm32(1), valuePayloadGPR);
3143             m_jit.store32(
3144                 valuePayloadGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3145             m_jit.load32(
3146                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)),
3147                 valueTagGPR);
3148             MacroAssembler::Jump slowCase = m_jit.branch32(MacroAssembler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
3149             m_jit.store32(
3150                 MacroAssembler::TrustedImm32(JSValue::EmptyValueTag),
3151                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3152             m_jit.load32(
3153                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)),
3154                 valuePayloadGPR);
3155
3156             addSlowPathGenerator(
3157                 slowPathMove(
3158                     undefinedCase, this,
3159                     MacroAssembler::TrustedImm32(jsUndefined().tag()), valueTagGPR,
3160                     MacroAssembler::TrustedImm32(jsUndefined().payload()), valuePayloadGPR));
3161             addSlowPathGenerator(
3162                 slowPathCall(
3163                     slowCase, this, operationArrayPopAndRecoverLength,
3164                     JSValueRegs(valueTagGPR, valuePayloadGPR), baseGPR));
3165
3166             jsValueResult(valueTagGPR, valuePayloadGPR, node);
3167             break;
3168         }
3169
3170         case Array::Double: {
3171             FPRTemporary temp(this);
3172             FPRReg tempFPR = temp.fpr();
3173
3174             m_jit.load32(
3175                 MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), valuePayloadGPR);
3176             MacroAssembler::Jump undefinedCase =
3177                 m_jit.branchTest32(MacroAssembler::Zero, valuePayloadGPR);
3178             m_jit.sub32(TrustedImm32(1), valuePayloadGPR);
3179             m_jit.store32(
3180                 valuePayloadGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3181             m_jit.loadDouble(
3182                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight),
3183                 tempFPR);
3184             MacroAssembler::Jump slowCase = m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempFPR, tempFPR);
3185             JSValue nan = JSValue(JSValue::EncodeAsDouble, QNaN);
3186             m_jit.store32(
3187                 MacroAssembler::TrustedImm32(nan.u.asBits.tag),
3188                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3189             m_jit.store32(
3190                 MacroAssembler::TrustedImm32(nan.u.asBits.payload),
3191                 MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3192             boxDouble(tempFPR, valueTagGPR, valuePayloadGPR);
3193
3194             addSlowPathGenerator(
3195                 slowPathMove(
3196                     undefinedCase, this,
3197                     MacroAssembler::TrustedImm32(jsUndefined().tag()), valueTagGPR,
3198                     MacroAssembler::TrustedImm32(jsUndefined().payload()), valuePayloadGPR));
3199             addSlowPathGenerator(
3200                 slowPathCall(
3201                     slowCase, this, operationArrayPopAndRecoverLength,
3202                     JSValueRegs(valueTagGPR, valuePayloadGPR), baseGPR));
3203
3204             jsValueResult(valueTagGPR, valuePayloadGPR, node);
3205             break;
3206         }
3207
3208         case Array::ArrayStorage: {
3209             GPRTemporary storageLength(this);
3210             GPRReg storageLengthGPR = storageLength.gpr();
3211
3212             m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR);
3213
3214             JITCompiler::JumpList setUndefinedCases;
3215             setUndefinedCases.append(m_jit.branchTest32(MacroAssembler::Zero, storageLengthGPR));
3216
3217             m_jit.sub32(TrustedImm32(1), storageLengthGPR);
3218
3219             MacroAssembler::Jump slowCase = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()));
3220
3221             m_jit.load32(MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), valueTagGPR);
3222             m_jit.load32(MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), valuePayloadGPR);
3223
3224             m_jit.store32(storageLengthGPR, MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()));
3225
3226             setUndefinedCases.append(m_jit.branch32(MacroAssembler::Equal, TrustedImm32(JSValue::EmptyValueTag), valueTagGPR));
3227
3228             m_jit.store32(TrustedImm32(JSValue::EmptyValueTag), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(ArrayStorage, m_vector[0]) + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3229
3230             m_jit.sub32(TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector)));
3231
3232             addSlowPathGenerator(
3233                 slowPathMove(
3234                     setUndefinedCases, this,
3235                     MacroAssembler::TrustedImm32(jsUndefined().tag()), valueTagGPR,
3236                     MacroAssembler::TrustedImm32(jsUndefined().payload()), valuePayloadGPR));
3237
3238             addSlowPathGenerator(
3239                 slowPathCall(
3240                     slowCase, this, operationArrayPop,
3241                     JSValueRegs(valueTagGPR, valuePayloadGPR), baseGPR));
3242
3243             jsValueResult(valueTagGPR, valuePayloadGPR, node);
3244             break;
3245         }
3246
3247         default:
3248             CRASH();
3249             break;
3250         }
3251         break;
3252     }
3253
3254     case DFG::Jump: {
3255         BlockIndex taken = node->takenBlockIndex();
3256         jump(taken);
3257         noResult(node);
3258         break;
3259     }
3260
3261     case Branch:
3262         emitBranch(node);
3263         break;
3264
3265     case Return: {
3266         ASSERT(GPRInfo::callFrameRegister != GPRInfo::regT2);
3267         ASSERT(GPRInfo::regT1 != GPRInfo::returnValueGPR);
3268         ASSERT(GPRInfo::returnValueGPR != GPRInfo::callFrameRegister);
3269
3270 #if DFG_ENABLE(SUCCESS_STATS)
3271         static SamplingCounter counter("SpeculativeJIT");
3272         m_jit.emitCount(counter);
3273 #endif
3274
3275         // Return the result in returnValueGPR.
3276         JSValueOperand op1(this, node->child1());
3277         op1.fill();
3278         if (op1.isDouble())
3279             boxDouble(op1.fpr(), GPRInfo::returnValueGPR2, GPRInfo::returnValueGPR);
3280         else {
3281             if (op1.payloadGPR() == GPRInfo::returnValueGPR2 && op1.tagGPR() == GPRInfo::returnValueGPR)
3282                 m_jit.swap(GPRInfo::returnValueGPR, GPRInfo::returnValueGPR2);
3283             else if (op1.payloadGPR() == GPRInfo::returnValueGPR2) {
3284                 m_jit.move(op1.payloadGPR(), GPRInfo::returnValueGPR);
3285                 m_jit.move(op1.tagGPR(), GPRInfo::returnValueGPR2);
3286             } else {
3287                 m_jit.move(op1.tagGPR(), GPRInfo::returnValueGPR2);
3288                 m_jit.move(op1.payloadGPR(), GPRInfo::returnValueGPR);
3289             }
3290         }
3291
3292         // Grab the return address.
3293         m_jit.emitGetFromCallFrameHeaderPtr(JSStack::ReturnPC, GPRInfo::regT2);
3294         // Restore our caller's "r".
3295         m_jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, GPRInfo::callFrameRegister);
3296         // Return.
3297         m_jit.restoreReturnAddressBeforeReturn(GPRInfo::regT2);
3298         m_jit.ret();
3299
3300         noResult(node);
3301         break;
3302     }
3303
3304     case Throw:
3305     case ThrowReferenceError: {
3306         // We expect that throw statements are rare and are intended to exit the code block
3307         // anyway, so we just OSR back to the old JIT for now.
3308         terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0);
3309         break;
3310     }
3311
3312     case ToPrimitive: {
3313         RELEASE_ASSERT(node->child1().useKind() == UntypedUse);
3314         JSValueOperand op1(this, node->child1());
3315         GPRTemporary resultTag(this, op1);
3316         GPRTemporary resultPayload(this, op1, false);
3317
3318         GPRReg op1TagGPR = op1.tagGPR();
3319         GPRReg op1PayloadGPR = op1.payloadGPR();
3320         GPRReg resultTagGPR = resultTag.gpr();
3321         GPRReg resultPayloadGPR = resultPayload.gpr();
3322
3323         op1.use();
3324
3325         if (!(m_state.forNode(node->child1()).m_type & ~(SpecNumber | SpecBoolean))) {
3326             m_jit.move(op1TagGPR, resultTagGPR);
3327             m_jit.move(op1PayloadGPR, resultPayloadGPR);
3328         } else {
3329             MacroAssembler::Jump alreadyPrimitive = m_jit.branch32(MacroAssembler::NotEqual, op1TagGPR, TrustedImm32(JSValue::CellTag));
3330             MacroAssembler::Jump notPrimitive = m_jit.branchPtr(MacroAssembler::NotEqual, MacroAssembler::Address(op1PayloadGPR, JSCell::structureOffset()), MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()));
3331
3332             alreadyPrimitive.link(&m_jit);
3333             m_jit.move(op1TagGPR, resultTagGPR);
3334             m_jit.move(op1PayloadGPR, resultPayloadGPR);
3335
3336             addSlowPathGenerator(
3337                 slowPathCall(
3338                     notPrimitive, this, operationToPrimitive,
3339                     JSValueRegs(resultTagGPR, resultPayloadGPR), op1TagGPR, op1PayloadGPR));
3340         }
3341
3342         jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3343         break;
3344     }
3345
3346     case ToString: {
3347         if (node->child1().useKind() == UntypedUse) {
3348             JSValueOperand op1(this, node->child1());
3349             GPRReg op1PayloadGPR = op1.payloadGPR();
3350             GPRReg op1TagGPR = op1.tagGPR();
3351
3352             GPRResult result(this);
3353             GPRReg resultGPR = result.gpr();
3354
3355             flushRegisters();
3356
3357             JITCompiler::Jump done;
3358             if (node->child1()->prediction() & SpecString) {
3359                 JITCompiler::Jump slowPath1 = m_jit.branch32(
3360                     JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::CellTag));
3361                 JITCompiler::Jump slowPath2 = m_jit.branchPtr(
3362                     JITCompiler::NotEqual,
3363                     JITCompiler::Address(op1PayloadGPR, JSCell::structureOffset()),
3364                     TrustedImmPtr(m_jit.vm()->stringStructure.get()));
3365                 m_jit.move(op1PayloadGPR, resultGPR);
3366                 done = m_jit.jump();
3367                 slowPath1.link(&m_jit);
3368                 slowPath2.link(&m_jit);
3369             }
3370             callOperation(operationToString, resultGPR, op1TagGPR, op1PayloadGPR);
3371             if (done.isSet())
3372                 done.link(&m_jit);
3373             cellResult(resultGPR, node);
3374             break;
3375         }
3376
3377         compileToStringOnCell(node);
3378         break;
3379     }
3380
3381     case NewStringObject: {
3382         compileNewStringObject(node);
3383         break;
3384     }
3385
3386     case NewArray: {
3387         JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin);
3388         if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) {
3389             globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint());
3390
3391             Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType());
3392             ASSERT(structure->indexingType() == node->indexingType());
3393             ASSERT(
3394                 hasUndecided(structure->indexingType())
3395                 || hasInt32(structure->indexingType())
3396                 || hasDouble(structure->indexingType())
3397                 || hasContiguous(structure->indexingType()));
3398
3399             unsigned numElements = node->numChildren();
3400
3401             GPRTemporary result(this);
3402             GPRTemporary storage(this);
3403
3404             GPRReg resultGPR = result.gpr();
3405             GPRReg storageGPR = storage.gpr();
3406
3407             emitAllocateJSArray(resultGPR, structure, storageGPR, numElements);
3408
3409             // At this point, one way or another, resultGPR and storageGPR have pointers to
3410             // the JSArray and the Butterfly, respectively.
3411
3412             ASSERT(!hasUndecided(structure->indexingType()) || !node->numChildren());
3413
3414             for (unsigned operandIdx = 0; operandIdx < node->numChildren(); ++operandIdx) {
3415                 Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx];
3416                 switch (node->indexingType()) {
3417                 case ALL_BLANK_INDEXING_TYPES:
3418                 case ALL_UNDECIDED_INDEXING_TYPES:
3419                     CRASH();
3420                     break;
3421                 case ALL_DOUBLE_INDEXING_TYPES: {
3422                     SpeculateDoubleOperand operand(this, use);
3423                     FPRReg opFPR = operand.fpr();
3424                     DFG_TYPE_CHECK(
3425                         JSValueRegs(), use, SpecRealNumber,
3426                         m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR));
3427
3428                     m_jit.storeDouble(opFPR, MacroAssembler::Address(storageGPR, sizeof(double) * operandIdx));
3429                     break;
3430                 }
3431                 case ALL_INT32_INDEXING_TYPES: {
3432                     SpeculateIntegerOperand operand(this, use);
3433                     m_jit.store32(TrustedImm32(JSValue::Int32Tag), MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3434                     m_jit.store32(operand.gpr(), MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3435                     break;
3436                 }
3437                 case ALL_CONTIGUOUS_INDEXING_TYPES: {
3438                     JSValueOperand operand(this, m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx]);
3439                     GPRReg opTagGPR = operand.tagGPR();
3440                     GPRReg opPayloadGPR = operand.payloadGPR();
3441                     m_jit.store32(opTagGPR, MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx + OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3442                     m_jit.store32(opPayloadGPR, MacroAssembler::Address(storageGPR, sizeof(JSValue) * operandIdx + OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3443                     break;
3444                 }
3445                 default:
3446                     CRASH();
3447                     break;
3448                 }
3449             }
3450
3451             // Yuck, we should *really* have a way of also returning the storageGPR. But
3452             // that's the least of what's wrong with this code. We really shouldn't be
3453             // allocating the array after having computed - and probably spilled to the
3454             // stack - all of the things that will go into the array. The solution to that
3455             // bigger problem will also likely fix the redundancy in reloading the storage
3456             // pointer that we currently have.
3457
3458             cellResult(resultGPR, node);
3459             break;
3460         }
3461
3462         if (!node->numChildren()) {
3463             flushRegisters();
3464             GPRResult result(this);
3465             callOperation(
3466                 operationNewEmptyArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()));
3467             cellResult(result.gpr(), node);
3468             break;
3469         }
3470
3471         size_t scratchSize = sizeof(EncodedJSValue) * node->numChildren();
3472         ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(scratchSize);
3473         EncodedJSValue* buffer = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
3474
3475         for (unsigned operandIdx = 0; operandIdx < node->numChildren(); ++operandIdx) {
3476             // Need to perform the speculations that this node promises to perform. If we're
3477             // emitting code here and the indexing type is not array storage then there is
3478             // probably something hilarious going on and we're already failing at all the
3479             // things, but at least we're going to be sound.
3480             Edge use = m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx];
3481             switch (node->indexingType()) {
3482             case ALL_BLANK_INDEXING_TYPES:
3483             case ALL_UNDECIDED_INDEXING_TYPES:
3484                 CRASH();
3485                 break;
3486             case ALL_DOUBLE_INDEXING_TYPES: {
3487                 SpeculateDoubleOperand operand(this, use);
3488                 FPRReg opFPR = operand.fpr();
3489                 DFG_TYPE_CHECK(
3490                     JSValueRegs(), use, SpecRealNumber,
3491                     m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR));
3492
3493                 m_jit.storeDouble(opFPR, reinterpret_cast<char*>(buffer + operandIdx));
3494                 break;
3495             }
3496             case ALL_INT32_INDEXING_TYPES: {
3497                 SpeculateIntegerOperand operand(this, use);
3498                 GPRReg opGPR = operand.gpr();
3499                 m_jit.store32(TrustedImm32(JSValue::Int32Tag), reinterpret_cast<char*>(buffer + operandIdx) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
3500                 m_jit.store32(opGPR, reinterpret_cast<char*>(buffer + operandIdx) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
3501                 break;
3502             }
3503             case ALL_CONTIGUOUS_INDEXING_TYPES:
3504             case ALL_ARRAY_STORAGE_INDEXING_TYPES: {
3505                 JSValueOperand operand(this, m_jit.graph().m_varArgChildren[node->firstChild() + operandIdx]);
3506                 GPRReg opTagGPR = operand.tagGPR();
3507                 GPRReg opPayloadGPR = operand.payloadGPR();
3508
3509                 m_jit.store32(opTagGPR, reinterpret_cast<char*>(buffer + operandIdx) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag));
3510                 m_jit.store32(opPayloadGPR, reinterpret_cast<char*>(buffer + operandIdx) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload));
3511                 operand.use();
3512                 break;
3513             }
3514             default:
3515                 CRASH();
3516                 break;
3517             }
3518         }
3519
3520         switch (node->indexingType()) {
3521         case ALL_DOUBLE_INDEXING_TYPES:
3522         case ALL_INT32_INDEXING_TYPES:
3523             useChildren(node);
3524             break;
3525         default:
3526             break;
3527         }
3528
3529         flushRegisters();
3530
3531         if (scratchSize) {
3532             GPRTemporary scratch(this);
3533
3534             // Tell GC mark phase how much of the scratch buffer is active during call.
3535             m_jit.move(TrustedImmPtr(scratchBuffer->activeLengthPtr()), scratch.gpr());
3536             m_jit.storePtr(TrustedImmPtr(scratchSize), scratch.gpr());
3537         }
3538
3539         GPRResult result(this);
3540
3541         callOperation(
3542             operationNewArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()),
3543             static_cast<void*>(buffer), node->numChildren());
3544
3545         if (scratchSize) {
3546             GPRTemporary scratch(this);
3547
3548             m_jit.move(TrustedImmPtr(scratchBuffer->activeLengthPtr()), scratch.gpr());
3549             m_jit.storePtr(TrustedImmPtr(0), scratch.gpr());
3550         }
3551
3552         cellResult(result.gpr(), node, UseChildrenCalledExplicitly);
3553         break;
3554     }
3555
3556     case NewArrayWithSize: {
3557         JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin);
3558         if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) {
3559             globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint());
3560
3561             SpeculateStrictInt32Operand size(this, node->child1());
3562             GPRTemporary result(this);
3563             GPRTemporary storage(this);
3564             GPRTemporary scratch(this);
3565             GPRTemporary scratch2(this);
3566
3567             GPRReg sizeGPR = size.gpr();
3568             GPRReg resultGPR = result.gpr();
3569             GPRReg storageGPR = storage.gpr();
3570             GPRReg scratchGPR = scratch.gpr();
3571             GPRReg scratch2GPR = scratch2.gpr();
3572
3573             MacroAssembler::JumpList slowCases;
3574             slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX)));
3575
3576             ASSERT((1 << 3) == sizeof(JSValue));
3577             m_jit.move(sizeGPR, scratchGPR);
3578             m_jit.lshift32(TrustedImm32(3), scratchGPR);
3579             m_jit.add32(TrustedImm32(sizeof(IndexingHeader)), scratchGPR, resultGPR);
3580             slowCases.append(
3581                 emitAllocateBasicStorage(resultGPR, storageGPR));
3582             m_jit.subPtr(scratchGPR, storageGPR);
3583             Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType());
3584             emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases);
3585
3586             m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()));
3587             m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength()));
3588
3589             if (hasDouble(node->indexingType())) {
3590                 JSValue nan = JSValue(JSValue::EncodeAsDouble, QNaN);
3591
3592                 m_jit.move(sizeGPR, scratchGPR);
3593                 MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratchGPR);
3594                 MacroAssembler::Label loop = m_jit.label();
3595                 m_jit.sub32(TrustedImm32(1), scratchGPR);
3596                 m_jit.store32(TrustedImm32(nan.u.asBits.tag), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)));
3597                 m_jit.store32(TrustedImm32(nan.u.asBits.payload), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)));
3598                 m_jit.branchTest32(MacroAssembler::NonZero, scratchGPR).linkTo(loop, &m_jit);
3599                 done.link(&m_jit);
3600             }
3601
3602             addSlowPathGenerator(adoptPtr(
3603                 new CallArrayAllocatorWithVariableSizeSlowPathGenerator(
3604                     slowCases, this, operationNewArrayWithSize, resultGPR,
3605                     globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()),
3606                     globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage),
3607                     sizeGPR)));
3608
3609             cellResult(resultGPR, node);
3610             break;
3611         }
3612
3613         SpeculateStrictInt32Operand size(this, node->child1());
3614         GPRReg sizeGPR = size.gpr();
3615         flushRegisters();
3616         GPRResult result(this);
3617         GPRReg resultGPR = result.gpr();
3618         GPRReg structureGPR = selectScratchGPR(sizeGPR);
3619         MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX));
3620         m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), structureGPR);
3621         MacroAssembler::Jump done = m_jit.jump();
3622         bigLength.link(&m_jit);
3623         m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)), structureGPR);
3624         done.link(&m_jit);
3625         callOperation(
3626             operationNewArrayWithSize, resultGPR, structureGPR, sizeGPR);
3627         cellResult(resultGPR, node);
3628         break;
3629     }
3630
3631     case NewArrayBuffer: {
3632         JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin);
3633         IndexingType indexingType = node->indexingType();
3634         if (!globalObject->isHavingABadTime() && !hasArrayStorage(indexingType)) {
3635             globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint());
3636
3637             unsigned numElements = node->numConstants();
3638
3639             GPRTemporary result(this);
3640             GPRTemporary storage(this);
3641
3642             GPRReg resultGPR = result.gpr();
3643             GPRReg storageGPR = storage.gpr();
3644
3645             emitAllocateJSArray(resultGPR, globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType), storageGPR, numElements);
3646
3647             if (node->indexingType() == ArrayWithDouble) {
3648                 JSValue* data = m_jit.codeBlock()->constantBuffer(node->startConstant());
3649                 for (unsigned index = 0; index < node->numConstants(); ++index) {
3650                     union {
3651                         int32_t halves[2];
3652                         double value;
3653                     } u;
3654                     u.value = data[index].asNumber();
3655                     m_jit.store32(Imm32(u.halves[0]), MacroAssembler::Address(storageGPR, sizeof(double) * index));
3656                     m_jit.store32(Imm32(u.halves[1]), MacroAssembler::Address(storageGPR, sizeof(double) * index + sizeof(int32_t)));
3657                 }
3658             } else {
3659                 int32_t* data = bitwise_cast<int32_t*>(m_jit.codeBlock()->constantBuffer(node->startConstant()));
3660                 for (unsigned index = 0; index < node->numConstants() * 2; ++index) {
3661                     m_jit.store32(
3662                         Imm32(data[index]), MacroAssembler::Address(storageGPR, sizeof(int32_t) * index));
3663                 }
3664             }
3665
3666             cellResult(resultGPR, node);
3667             break;
3668         }
3669
3670         flushRegisters();
3671         GPRResult result(this);
3672
3673         callOperation(operationNewArrayBuffer, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), node->startConstant(), node->numConstants());
3674
3675         cellResult(result.gpr(), node);
3676         break;
3677     }
3678
3679     case NewRegexp: {
3680         flushRegisters();
3681         GPRResult resultPayload(this);
3682         GPRResult2 resultTag(this);
3683
3684         callOperation(operationNewRegexp, resultTag.gpr(), resultPayload.gpr(), m_jit.codeBlock()->regexp(node->regexpIndex()));
3685
3686         // FIXME: make the callOperation above explicitly return a cell result, or jitAssert the tag is a cell tag.
3687         cellResult(resultPayload.gpr(), node);
3688         break;
3689     }
3690
3691     case ConvertThis: {
3692         ASSERT(node->child1().useKind() == UntypedUse);
3693
3694         JSValueOperand thisValue(this, node->child1());
3695         GPRReg thisValueTagGPR = thisValue.tagGPR();
3696         GPRReg thisValuePayloadGPR = thisValue.payloadGPR();
3697
3698         flushRegisters();
3699
3700         GPRResult2 resultTag(this);
3701         GPRResult resultPayload(this);
3702         callOperation(operationConvertThis, resultTag.gpr(), resultPayload.gpr(), thisValueTagGPR, thisValuePayloadGPR);
3703
3704         cellResult(resultPayload.gpr(), node);
3705         break;
3706     }
3707
3708     case CreateThis: {
3709         // Note that there is not so much profit to speculate here. The only things we
3710         // speculate on are (1) that it's a cell, since that eliminates cell checks
3711         // later if the proto is reused, and (2) if we have a FinalObject prediction
3712         // then we speculate because we want to get recompiled if it isn't (since
3713         // otherwise we'd start taking slow path a lot).
3714
3715         SpeculateCellOperand callee(this, node->child1());
3716         GPRTemporary result(this);
3717         GPRTemporary allocator(this);
3718         GPRTemporary structure(this);
3719         GPRTemporary scratch(this);
3720
3721         GPRReg calleeGPR = callee.gpr();
3722         GPRReg resultGPR = result.gpr();
3723         GPRReg allocatorGPR = allocator.gpr();
3724         GPRReg structureGPR = structure.gpr();
3725         GPRReg scratchGPR = scratch.gpr();
3726
3727         MacroAssembler::JumpList slowPath;
3728
3729         m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR);
3730         m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR);
3731         slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, allocatorGPR));
3732         emitAllocateJSObject(resultGPR, allocatorGPR, structureGPR, TrustedImmPtr(0), scratchGPR, slowPath);
3733
3734         addSlowPathGenerator(slowPathCall(slowPath, this, operationCreateThis, resultGPR, calleeGPR, node->inlineCapacity()));
3735
3736         cellResult(resultGPR, node);
3737         break;
3738     }
3739
3740     case AllocationProfileWatchpoint: {
3741         jsCast<JSFunction*>(node->function())->addAllocationProfileWatchpoint(speculationWatchpoint());
3742         noResult(node);
3743         break;
3744     }
3745
3746     case NewObject: {
3747         GPRTemporary result(this);
3748         GPRTemporary allocator(this);
3749         GPRTemporary scratch(this);
3750
3751         GPRReg resultGPR = result.gpr();
3752         GPRReg allocatorGPR = allocator.gpr();
3753         GPRReg scratchGPR = scratch.gpr();
3754
3755         MacroAssembler::JumpList slowPath;
3756
3757         Structure* structure = node->structure();
3758         size_t allocationSize = JSObject::allocationSize(structure->inlineCapacity());
3759         MarkedAllocator* allocatorPtr = &m_jit.vm()->heap.allocatorForObjectWithoutDestructor(allocationSize);
3760
3761         m_jit.move(TrustedImmPtr(allocatorPtr), allocatorGPR);
3762         emitAllocateJSObject(resultGPR, allocatorGPR, TrustedImmPtr(structure), TrustedImmPtr(0), scratchGPR, slowPath);
3763
3764         addSlowPathGenerator(slowPathCall(slowPath, this, operationNewObject, resultGPR, structure));
3765
3766         cellResult(resultGPR, node);
3767         break;
3768     }
3769
3770     case GetCallee: {
3771         GPRTemporary result(this);
3772         m_jit.loadPtr(JITCompiler::payloadFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::Callee))), result.gpr());
3773         cellResult(result.gpr(), node);
3774         break;
3775     }
3776
3777     case SetCallee: {
3778         SpeculateCellOperand callee(this, node->child1());
3779         m_jit.storePtr(callee.gpr(), JITCompiler::payloadFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::Callee))));
3780         m_jit.store32(MacroAssembler::TrustedImm32(JSValue::CellTag), JITCompiler::tagFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::Callee))));
3781         noResult(node);
3782         break;
3783     }
3784
3785     case GetScope: {
3786         SpeculateCellOperand function(this, node->child1());
3787         GPRTemporary result(this, function);
3788         m_jit.loadPtr(JITCompiler::Address(function.gpr(), JSFunction::offsetOfScopeChain()), result.gpr());
3789         cellResult(result.gpr(), node);
3790         break;
3791     }
3792
3793     case GetMyScope: {
3794         GPRTemporary result(this);
3795         GPRReg resultGPR = result.gpr();
3796
3797         m_jit.loadPtr(JITCompiler::payloadFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::ScopeChain))), resultGPR);
3798         cellResult(resultGPR, node);
3799         break;
3800     }
3801
3802     case SetMyScope: {
3803         SpeculateCellOperand callee(this, node->child1());
3804         m_jit.storePtr(callee.gpr(), JITCompiler::payloadFor(static_cast<VirtualRegister>(node->codeOrigin.stackOffset() + static_cast<int>(JSStack::ScopeChain))));
3805         noResult(node);
3806         break;
3807     }
3808
3809     case SkipTopScope: {
3810         SpeculateCellOperand scope(this, node->child1());
3811         GPRTemporary result(this, scope);
3812         GPRReg resultGPR = result.gpr();
3813         m_jit.move(scope.gpr(), resultGPR);
3814         JITCompiler::Jump activationNotCreated =
3815             m_jit.branchTestPtr(
3816                 JITCompiler::Zero,
3817                 JITCompiler::payloadFor(
3818                     static_cast<VirtualRegister>(m_jit.codeBlock()->activationRegister())));
3819         m_jit.loadPtr(JITCompiler::Address(resultGPR, JSScope::offsetOfNext()), resultGPR);
3820         activationNotCreated.link(&m_jit);
3821         cellResult(resultGPR, node);
3822         break;
3823     }
3824
3825     case SkipScope: {
3826         SpeculateCellOperand scope(this, node->child1());
3827         GPRTemporary result(this, scope);
3828         m_jit.loadPtr(JITCompiler::Address(scope.gpr(), JSScope::offsetOfNext()), result.gpr());
3829         cellResult(result.gpr(), node);
3830         break;
3831     }
3832
3833     case GetScopeRegisters: {
3834         SpeculateCellOperand scope(this, node->child1());
3835         GPRTemporary result(this);
3836         GPRReg scopeGPR = scope.gpr();
3837         GPRReg resultGPR = result.gpr();
3838
3839         m_jit.loadPtr(JITCompiler::Address(scopeGPR, JSVariableObject::offsetOfRegisters()), resultGPR);
3840         storageResult(resultGPR, node);
3841         break;
3842     }
3843     case GetScopedVar: {
3844         StorageOperand registers(this, node->child1());
3845         GPRTemporary resultTag(this);
3846         GPRTemporary resultPayload(this);
3847         GPRReg registersGPR = registers.gpr();
3848         GPRReg resultTagGPR = resultTag.gpr();
3849         GPRReg resultPayloadGPR = resultPayload.gpr();
3850         m_jit.load32(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTagGPR);
3851         m_jit.load32(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR);
3852         jsValueResult(resultTagGPR, resultPayloadGPR, node);
3853         break;
3854     }
3855     case PutScopedVar: {
3856         SpeculateCellOperand scope(this, node->child1());
3857         StorageOperand registers(this, node->child2());
3858         JSValueOperand value(this, node->child3());
3859         GPRTemporary scratchRegister(this);
3860         GPRReg scopeGPR = scope.gpr();
3861         GPRReg registersGPR = registers.gpr();
3862         GPRReg valueTagGPR = value.tagGPR();
3863         GPRReg valuePayloadGPR = value.payloadGPR();
3864         GPRReg scratchGPR = scratchRegister.gpr();
3865
3866         m_jit.store32(valueTagGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
3867         m_jit.store32(valuePayloadGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
3868         writeBarrier(scopeGPR, valueTagGPR, node->child2(), WriteBarrierForVariableAccess, scratchGPR);
3869         noResult(node);
3870         break;
3871     }
3872
3873     case GetById: {
3874         if (!node->prediction()) {
3875             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
3876             break;
3877         }
3878
3879         if (isCellSpeculation(node->child1()->prediction())) {
3880             SpeculateCellOperand base(this, node->child1());
3881             GPRTemporary resultTag(this, base);
3882             GPRTemporary resultPayload(this);
3883
3884             GPRReg baseGPR = base.gpr();
3885             GPRReg resultTagGPR = resultTag.gpr();
3886             GPRReg resultPayloadGPR = resultPayload.gpr();
3887
3888             base.use();
3889
3890             cachedGetById(node->codeOrigin, InvalidGPRReg, baseGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber());
3891
3892             jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3893             break;
3894         }
3895
3896         JSValueOperand base(this, node->child1());
3897         GPRTemporary resultTag(this, base);
3898         GPRTemporary resultPayload(this);
3899
3900         GPRReg baseTagGPR = base.tagGPR();
3901         GPRReg basePayloadGPR = base.payloadGPR();
3902         GPRReg resultTagGPR = resultTag.gpr();
3903         GPRReg resultPayloadGPR = resultPayload.gpr();
3904
3905         base.use();
3906
3907         JITCompiler::Jump notCell = m_jit.branch32(JITCompiler::NotEqual, baseTagGPR, TrustedImm32(JSValue::CellTag));
3908
3909         cachedGetById(node->codeOrigin, baseTagGPR, basePayloadGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), notCell);
3910
3911         jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3912         break;
3913     }
3914
3915     case GetByIdFlush: {
3916         if (!node->prediction()) {
3917             terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
3918             break;
3919         }
3920
3921         switch (node->child1().useKind()) {
3922         case CellUse: {
3923             SpeculateCellOperand base(this, node->child1());
3924
3925             GPRReg baseGPR = base.gpr();
3926
3927             GPRResult resultTag(this);
3928             GPRResult2 resultPayload(this);
3929             GPRReg resultTagGPR = resultTag.gpr();
3930             GPRReg resultPayloadGPR = resultPayload.gpr();
3931
3932             base.use();
3933
3934             flushRegisters();
3935
3936             cachedGetById(node->codeOrigin, InvalidGPRReg, baseGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill);
3937
3938             jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3939             break;
3940         }
3941
3942         case UntypedUse: {
3943             JSValueOperand base(this, node->child1());
3944             GPRReg baseTagGPR = base.tagGPR();
3945             GPRReg basePayloadGPR = base.payloadGPR();
3946
3947             GPRResult resultTag(this);
3948             GPRResult2 resultPayload(this);
3949             GPRReg resultTagGPR = resultTag.gpr();
3950             GPRReg resultPayloadGPR = resultPayload.gpr();
3951
3952             base.use();
3953
3954             flushRegisters();
3955
3956             JITCompiler::Jump notCell = m_jit.branch32(JITCompiler::NotEqual, baseTagGPR, TrustedImm32(JSValue::CellTag));
3957
3958             cachedGetById(node->codeOrigin, baseTagGPR, basePayloadGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), notCell, DontSpill);
3959
3960             jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly);
3961             break;
3962         }
3963
3964         default:
3965             RELEASE_ASSERT_NOT_REACHED();
3966             break;
3967         }
3968         break;
3969     }
3970
3971     case GetArrayLength:
3972         compileGetArrayLength(node);
3973         break;
3974
3975     case CheckFunction: {
3976         SpeculateCellOperand function(this, node->child1());
3977         speculationCheck(BadFunction, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, function.gpr(), node->function()));
3978         noResult(node);
3979         break;
3980     }
3981
3982     case CheckExecutable: {
3983         SpeculateCellOperand function(this, node->child1());
3984         speculationCheck(BadExecutable, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, JITCompiler::Address(function.gpr(), JSFunction::offsetOfExecutable()), node->executable()));
3985         noResult(node);
3986         break;
3987     }
3988
3989     case CheckStructure:
3990     case ForwardCheckStructure: {
3991         SpeculateCellOperand base(this, node->child1());
3992
3993         ASSERT(node->structureSet().size());
3994
3995         if (node->structureSet().size() == 1) {
3996             speculationCheck(
3997                 BadCache, JSValueSource::unboxedCell(base.gpr()), 0,
3998                 m_jit.branchWeakPtr(
3999                     JITCompiler::NotEqual,
4000                     JITCompiler::Address(base.gpr(), JSCell::structureOffset()),
4001                     node->structureSet()[0]));
4002         } else {
4003             GPRTemporary structure(this);
4004
4005             m_jit.loadPtr(JITCompiler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr());
4006
4007             JITCompiler::JumpList done;
4008
4009             for (size_t i = 0; i < node->structureSet().size() - 1; ++i)
4010                 done.append(m_jit.branchWeakPtr(JITCompiler::Equal, structure.gpr(), node->structureSet()[i]));
4011
4012             speculationCheck(
4013                 BadCache, JSValueSource::unboxedCell(base.gpr()), 0,
4014                 m_jit.branchWeakPtr(
4015                     JITCompiler::NotEqual, structure.gpr(), node->structureSet().last()));
4016
4017             done.link(&m_jit);
4018         }
4019
4020         noResult(node);
4021         break;
4022     }
4023
4024     case StructureTransitionWatchpoint:
4025     case ForwardStructureTransitionWatchpoint: {
4026         // There is a fascinating question here of what to do about array profiling.
4027         // We *could* try to tell the OSR exit about where the base of the access is.
4028         // The DFG will have kept it alive, though it may not be in a register, and
4029         // we shouldn't really load it since that could be a waste. For now though,
4030         // we'll just rely on the fact that when a watchpoint fires then that's
4031         // quite a hint already.
4032
4033         m_jit.addWeakReference(node->structure());
4034         node->structure()->addTransitionWatchpoint(
4035             speculationWatchpoint(
4036                 node->child1()->op() == WeakJSConstant ? BadWeakConstantCache : BadCache));
4037
4038 #if !ASSERT_DISABLED
4039         SpeculateCellOperand op1(this, node->child1());
4040         JITCompiler::Jump isOK = m_jit.branchPtr(JITCompiler::Equal, JITCompiler::Address(op1.gpr(), JSCell::structureOffset()), TrustedImmPtr(node->structure()));
4041         m_jit.breakpoint();
4042         isOK.link(&m_jit);
4043 #else
4044         speculateCell(node->child1());
4045 #endif
4046
4047         noResult(node);
4048         break;
4049     }
4050
4051     case PhantomPutStructure: {
4052         ASSERT(isKnownCell(node->child1().node()));
4053         ASSERT(node->structureTransitionData().previousStructure->transitionWatchpointSetHasBeenInvalidated());
4054         m_jit.addWeakReferenceTransition(
4055             node->codeOrigin.codeOriginOwner(),
4056             node->structureTransitionData().previousStructure,
4057             node->structureTransitionData().newStructure);
4058         noResult(node);
4059         break;
4060     }
4061
4062     case PutStructure: {
4063         ASSERT(node->structureTransitionData().previousStructure->transitionWatchpointSetHasBeenInvalidated());
4064
4065         SpeculateCellOperand base(this, node->child1());
4066         GPRReg baseGPR = base.gpr();
4067
4068         m_jit.addWeakReferenceTransition(
4069             node->codeOrigin.codeOriginOwner(),
4070             node->structureTransitionData().previousStructure,
4071             node->structureTransitionData().newStructure);
4072
4073 #if ENABLE(WRITE_BARRIER_PROFILING)
4074         // Must always emit this write barrier as the structure transition itself requires it
4075         writeBarrier(baseGPR, node->structureTransitionData().newStructure, WriteBarrierForGenericAccess);
4076 #endif
4077
4078         m_jit.storePtr(MacroAssembler::TrustedImmPtr(node->structureTransitionData().newStructure), MacroAssembler::Address(baseGPR, JSCell::structureOffset()));
4079
4080         noResult(node);
4081         break;
4082     }
4083
4084     case AllocatePropertyStorage:
4085         compileAllocatePropertyStorage(node);
4086         break;
4087
4088     case ReallocatePropertyStorage:
4089         compileReallocatePropertyStorage(node);
4090         break;
4091
4092     case GetButterfly: {
4093         SpeculateCellOperand base(this, node->child1());
4094         GPRTemporary result(this, base);
4095
4096         GPRReg baseGPR = base.gpr();
4097         GPRReg resultGPR = result.gpr();
4098
4099         m_jit.loadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), resultGPR);
4100
4101         storageResult(resultGPR, node);
4102         break;
4103     }
4104
4105     case GetIndexedPropertyStorage: {
4106         compileGetIndexedPropertyStorage(node);
4107         break;
4108     }
4109
4110     case GetByOffset: {
4111         StorageOperand storage(this, node->child1());
4112         GPRTemporary resultTag(this, storage);
4113         GPRTemporary resultPayload(this);
4114
4115         GPRReg storageGPR = storage.gpr();
4116         GPRReg resultTagGPR = resultTag.gpr();
4117         GPRReg resultPayloadGPR = resultPayload.gpr();
4118
4119         StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()];
4120
4121         m_jit.load32(JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR);
4122         m_jit.load32(JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTagGPR);
4123
4124         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4125         break;
4126     }
4127
4128     case PutByOffset: {
4129 #if ENABLE(WRITE_BARRIER_PROFILING)
4130         SpeculateCellOperand base(this, node->child2());
4131 #endif
4132         StorageOperand storage(this, node->child1());
4133         JSValueOperand value(this, node->child3());
4134
4135         GPRReg storageGPR = storage.gpr();
4136         GPRReg valueTagGPR = value.tagGPR();
4137         GPRReg valuePayloadGPR = value.payloadGPR();
4138
4139 #if ENABLE(WRITE_BARRIER_PROFILING)
4140         writeBarrier(base.gpr(), valueTagGPR, node->child3(), WriteBarrierForPropertyAccess);
4141 #endif
4142
4143         StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()];
4144
4145         m_jit.storePtr(valueTagGPR, JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
4146         m_jit.storePtr(valuePayloadGPR, JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
4147
4148         noResult(node);
4149         break;
4150     }
4151
4152     case PutById: {
4153         SpeculateCellOperand base(this, node->child1());
4154         JSValueOperand value(this, node->child2());
4155         GPRTemporary scratch(this);
4156
4157         GPRReg baseGPR = base.gpr();
4158         GPRReg valueTagGPR = value.tagGPR();
4159         GPRReg valuePayloadGPR = value.payloadGPR();
4160         GPRReg scratchGPR = scratch.gpr();
4161
4162         base.use();
4163         value.use();
4164
4165         cachedPutById(node->codeOrigin, baseGPR, valueTagGPR, valuePayloadGPR, node->child2(), scratchGPR, node->identifierNumber(), NotDirect);
4166
4167         noResult(node, UseChildrenCalledExplicitly);
4168         break;
4169     }
4170
4171     case PutByIdDirect: {
4172         SpeculateCellOperand base(this, node->child1());
4173         JSValueOperand value(this, node->child2());
4174         GPRTemporary scratch(this);
4175
4176         GPRReg baseGPR = base.gpr();
4177         GPRReg valueTagGPR = value.tagGPR();
4178         GPRReg valuePayloadGPR = value.payloadGPR();
4179         GPRReg scratchGPR = scratch.gpr();
4180
4181         base.use();
4182         value.use();
4183
4184         cachedPutById(node->codeOrigin, baseGPR, valueTagGPR, valuePayloadGPR, node->child2(), scratchGPR, node->identifierNumber(), Direct);
4185
4186         noResult(node, UseChildrenCalledExplicitly);
4187         break;
4188     }
4189
4190     case GetGlobalVar: {
4191         GPRTemporary resultPayload(this);
4192         GPRTemporary resultTag(this);
4193
4194         m_jit.move(TrustedImmPtr(node->registerPointer()), resultPayload.gpr());
4195         m_jit.load32(JITCompiler::Address(resultPayload.gpr(), OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTag.gpr());
4196         m_jit.load32(JITCompiler::Address(resultPayload.gpr(), OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayload.gpr());
4197
4198         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
4199         break;
4200     }
4201
4202     case PutGlobalVar: {
4203         JSValueOperand value(this, node->child1());
4204         if (Heap::isWriteBarrierEnabled()) {
4205             GPRTemporary scratch(this);
4206             GPRReg scratchReg = scratch.gpr();
4207
4208             writeBarrier(m_jit.globalObjectFor(node->codeOrigin), value.tagGPR(), node->child1(), WriteBarrierForVariableAccess, scratchReg);
4209         }
4210
4211         // FIXME: if we happen to have a spare register - and _ONLY_ if we happen to have
4212         // a spare register - a good optimization would be to put the register pointer into
4213         // a register and then do a zero offset store followed by a four-offset store (or
4214         // vice-versa depending on endianness).
4215         m_jit.store32(value.tagGPR(), node->registerPointer()->tagPointer());
4216         m_jit.store32(value.payloadGPR(), node->registerPointer()->payloadPointer());
4217
4218         noResult(node);
4219         break;
4220     }
4221
4222     case PutGlobalVarCheck: {
4223         JSValueOperand value(this, node->child1());
4224
4225         WatchpointSet* watchpointSet =
4226             m_jit.globalObjectFor(node->codeOrigin)->symbolTable()->get(
4227                 identifier(node->identifierNumberForCheck())->impl()).watchpointSet();
4228         addSlowPathGenerator(
4229             slowPathCall(
4230                 m_jit.branchTest8(
4231                     JITCompiler::NonZero,
4232                     JITCompiler::AbsoluteAddress(watchpointSet->addressOfIsWatched())),
4233                 this, operationNotifyGlobalVarWrite, NoResult, watchpointSet));
4234
4235         if (Heap::isWriteBarrierEnabled()) {
4236             GPRTemporary scratch(this);
4237             GPRReg scratchReg = scratch.gpr();
4238
4239             writeBarrier(m_jit.globalObjectFor(node->codeOrigin), value.tagGPR(), node->child1(), WriteBarrierForVariableAccess, scratchReg);
4240         }
4241
4242         // FIXME: if we happen to have a spare register - and _ONLY_ if we happen to have
4243         // a spare register - a good optimization would be to put the register pointer into
4244         // a register and then do a zero offset store followed by a four-offset store (or
4245         // vice-versa depending on endianness).
4246         m_jit.store32(value.tagGPR(), node->registerPointer()->tagPointer());
4247         m_jit.store32(value.payloadGPR(), node->registerPointer()->payloadPointer());
4248
4249         noResult(node);
4250         break;
4251     }
4252
4253     case GlobalVarWatchpoint: {
4254         m_jit.globalObjectFor(node->codeOrigin)->symbolTable()->get(
4255             identifier(node->identifierNumberForCheck())->impl()).addWatchpoint(
4256                 speculationWatchpoint());
4257
4258 #if DFG_ENABLE(JIT_ASSERT)
4259         GPRTemporary scratch(this);
4260         GPRReg scratchGPR = scratch.gpr();
4261         m_jit.load32(node->registerPointer()->tagPointer(), scratchGPR);
4262         JITCompiler::Jump notOK = m_jit.branch32(
4263             JITCompiler::NotEqual, scratchGPR,
4264             TrustedImm32(node->registerPointer()->get().tag()));
4265         m_jit.load32(node->registerPointer()->payloadPointer(), scratchGPR);
4266         JITCompiler::Jump ok = m_jit.branch32(
4267             JITCompiler::Equal, scratchGPR,
4268             TrustedImm32(node->registerPointer()->get().payload()));
4269         notOK.link(&m_jit);
4270         m_jit.breakpoint();
4271         ok.link(&m_jit);
4272 #endif
4273
4274         noResult(node);
4275         break;
4276     }
4277
4278     case CheckHasInstance: {
4279         SpeculateCellOperand base(this, node->child1());
4280         GPRTemporary structure(this);
4281
4282         // Speculate that base 'ImplementsDefaultHasInstance'.
4283         m_jit.loadPtr(MacroAssembler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr());
4284         speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(structure.gpr(), Structure::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance)));
4285
4286         noResult(node);
4287         break;
4288     }
4289
4290     case InstanceOf: {
4291         compileInstanceOf(node);
4292         break;
4293     }
4294
4295     case IsUndefined: {
4296         JSValueOperand value(this, node->child1());
4297         GPRTemporary result(this);
4298         GPRTemporary localGlobalObject(this);
4299         GPRTemporary remoteGlobalObject(this);
4300
4301         JITCompiler::Jump isCell = m_jit.branch32(JITCompiler::Equal, value.tagGPR(), JITCompiler::TrustedImm32(JSValue::CellTag));
4302
4303         m_jit.compare32(JITCompiler::Equal, value.tagGPR(), TrustedImm32(JSValue::UndefinedTag), result.gpr());
4304         JITCompiler::Jump done = m_jit.jump();
4305
4306         isCell.link(&m_jit);
4307         JITCompiler::Jump notMasqueradesAsUndefined;
4308         if (m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) {
4309             m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint());
4310             m_jit.move(TrustedImm32(0), result.gpr());
4311             notMasqueradesAsUndefined = m_jit.jump();
4312         } else {
4313             m_jit.loadPtr(JITCompiler::Address(value.payloadGPR(), JSCell::structureOffset()), result.gpr());
4314             JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(result.gpr(), Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined));
4315             m_jit.move(TrustedImm32(0), result.gpr());
4316             notMasqueradesAsUndefined = m_jit.jump();
4317
4318             isMasqueradesAsUndefined.link(&m_jit);
4319             GPRReg localGlobalObjectGPR = localGlobalObject.gpr();
4320             GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr();
4321             m_jit.move(TrustedImmPtr(m_jit.globalObjectFor(node->codeOrigin)), localGlobalObjectGPR);
4322             m_jit.loadPtr(JITCompiler::Address(result.gpr(), Structure::globalObjectOffset()), remoteGlobalObjectGPR);
4323             m_jit.compare32(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, result.gpr());
4324         }
4325
4326         notMasqueradesAsUndefined.link(&m_jit);
4327         done.link(&m_jit);
4328         booleanResult(result.gpr(), node);
4329         break;
4330     }
4331
4332     case IsBoolean: {
4333         JSValueOperand value(this, node->child1());
4334         GPRTemporary result(this, value);
4335
4336         m_jit.compare32(JITCompiler::Equal, value.tagGPR(), JITCompiler::TrustedImm32(JSValue::BooleanTag), result.gpr());
4337         booleanResult(result.gpr(), node);
4338         break;
4339     }
4340
4341     case IsNumber: {
4342         JSValueOperand value(this, node->child1());
4343         GPRTemporary result(this, value);
4344
4345         m_jit.add32(TrustedImm32(1), value.tagGPR(), result.gpr());
4346         m_jit.compare32(JITCompiler::Below, result.gpr(), JITCompiler::TrustedImm32(JSValue::LowestTag + 1), result.gpr());
4347         booleanResult(result.gpr(), node);
4348         break;
4349     }
4350
4351     case IsString: {
4352         JSValueOperand value(this, node->child1());
4353         GPRTemporary result(this, value);
4354
4355         JITCompiler::Jump isNotCell = m_jit.branch32(JITCompiler::NotEqual, value.tagGPR(), JITCompiler::TrustedImm32(JSValue::CellTag));
4356
4357         m_jit.loadPtr(JITCompiler::Address(value.payloadGPR(), JSCell::structureOffset()), result.gpr());
4358         m_jit.compare8(JITCompiler::Equal, JITCompiler::Address(result.gpr(), Structure::typeInfoTypeOffset()), TrustedImm32(StringType), result.gpr());
4359         JITCompiler::Jump done = m_jit.jump();
4360
4361         isNotCell.link(&m_jit);
4362         m_jit.move(TrustedImm32(0), result.gpr());
4363
4364         done.link(&m_jit);
4365         booleanResult(result.gpr(), node);
4366         break;
4367     }
4368
4369     case IsObject: {
4370         JSValueOperand value(this, node->child1());
4371         GPRReg valueTagGPR = value.tagGPR();
4372         GPRReg valuePayloadGPR = value.payloadGPR();
4373         GPRResult result(this);
4374         GPRReg resultGPR = result.gpr();
4375         flushRegisters();
4376         callOperation(operationIsObject, resultGPR, valueTagGPR, valuePayloadGPR);
4377         booleanResult(result.gpr(), node);
4378         break;
4379     }
4380
4381     case IsFunction: {
4382         JSValueOperand value(this, node->child1());
4383         GPRReg valueTagGPR = value.tagGPR();
4384         GPRReg valuePayloadGPR = value.payloadGPR();
4385         GPRResult result(this);
4386         GPRReg resultGPR = result.gpr();
4387         flushRegisters();
4388         callOperation(operationIsFunction, resultGPR, valueTagGPR, valuePayloadGPR);
4389         booleanResult(result.gpr(), node);
4390         break;
4391     }
4392     case TypeOf: {
4393         JSValueOperand value(this, node->child1(), ManualOperandSpeculation);
4394         GPRReg tagGPR = value.tagGPR();
4395         GPRReg payloadGPR = value.payloadGPR();
4396         GPRTemporary temp(this);
4397         GPRReg tempGPR = temp.gpr();
4398         GPRResult result(this);
4399         GPRReg resultGPR = result.gpr();
4400         JITCompiler::JumpList doneJumps;
4401
4402         flushRegisters();
4403
4404         ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == CellUse || node->child1().useKind() == StringUse);
4405
4406         JITCompiler::Jump isNotCell = m_jit.branch32(JITCompiler::NotEqual, tagGPR, JITCompiler::TrustedImm32(JSValue::CellTag));
4407         if (node->child1().useKind() != UntypedUse)
4408             DFG_TYPE_CHECK(JSValueRegs(tagGPR, payloadGPR), node->child1(), SpecCell, isNotCell);
4409
4410         if (!node->child1()->shouldSpeculateObject() || node->child1().useKind() == StringUse) {
4411             m_jit.loadPtr(JITCompiler::Address(payloadGPR, JSCell::structureOffset()), tempGPR);
4412             JITCompiler::Jump notString = m_jit.branch8(JITCompiler::NotEqual, JITCompiler::Address(tempGPR, Structure::typeInfoTypeOffset()), TrustedImm32(StringType));
4413             if (node->child1().useKind() == StringUse)
4414                 DFG_TYPE_CHECK(JSValueRegs(tagGPR, payloadGPR), node->child1(), SpecString, notString);
4415             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.stringString()), resultGPR);
4416             doneJumps.append(m_jit.jump());
4417             if (node->child1().useKind() != StringUse) {
4418                 notString.link(&m_jit);
4419                 callOperation(operationTypeOf, resultGPR, payloadGPR);
4420                 doneJumps.append(m_jit.jump());
4421             }
4422         } else {
4423             callOperation(operationTypeOf, resultGPR, payloadGPR);
4424             doneJumps.append(m_jit.jump());
4425         }
4426
4427         if (node->child1().useKind() == UntypedUse) {
4428             isNotCell.link(&m_jit);
4429
4430             m_jit.add32(TrustedImm32(1), tagGPR, tempGPR);
4431             JITCompiler::Jump notNumber = m_jit.branch32(JITCompiler::AboveOrEqual, tempGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1));
4432             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.numberString()), resultGPR);
4433             doneJumps.append(m_jit.jump());
4434             notNumber.link(&m_jit);
4435
4436             JITCompiler::Jump notUndefined = m_jit.branch32(JITCompiler::NotEqual, tagGPR, TrustedImm32(JSValue::UndefinedTag));
4437             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.undefinedString()), resultGPR);
4438             doneJumps.append(m_jit.jump());
4439             notUndefined.link(&m_jit);
4440
4441             JITCompiler::Jump notNull = m_jit.branch32(JITCompiler::NotEqual, tagGPR, TrustedImm32(JSValue::NullTag));
4442             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.objectString()), resultGPR);
4443             doneJumps.append(m_jit.jump());
4444             notNull.link(&m_jit);
4445
4446             // Only boolean left
4447             m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.booleanString()), resultGPR);
4448         }
4449         doneJumps.link(&m_jit);
4450         cellResult(resultGPR, node);
4451         break;
4452     }
4453
4454     case Phi:
4455     case Flush:
4456         break;
4457
4458     case Breakpoint:
4459 #if ENABLE(DEBUG_WITH_BREAKPOINT)
4460         m_jit.breakpoint();
4461 #else
4462         RELEASE_ASSERT_NOT_REACHED();
4463 #endif
4464         break;
4465
4466     case Call:
4467     case Construct:
4468         emitCall(node);
4469         break;
4470
4471     case Resolve: {
4472         flushRegisters();
4473         GPRResult resultPayload(this);
4474         GPRResult2 resultTag(this);
4475         ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()];
4476         callOperation(operationResolve, resultTag.gpr(), resultPayload.gpr(), identifier(data.identifierNumber), data.resolveOperations);
4477         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
4478         break;
4479     }
4480
4481     case ResolveBase: {
4482         flushRegisters();
4483         GPRResult resultPayload(this);
4484         GPRResult2 resultTag(this);
4485         ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()];
4486         callOperation(operationResolveBase, resultTag.gpr(), resultPayload.gpr(), identifier(data.identifierNumber), data.resolveOperations, data.putToBaseOperation);
4487         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
4488         break;
4489     }
4490
4491     case ResolveBaseStrictPut: {
4492         flushRegisters();
4493         GPRResult resultPayload(this);
4494         GPRResult2 resultTag(this);
4495         ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()];
4496         callOperation(operationResolveBaseStrictPut, resultTag.gpr(), resultPayload.gpr(), identifier(data.identifierNumber), data.resolveOperations, data.putToBaseOperation);
4497         jsValueResult(resultTag.gpr(), resultPayload.gpr(), node);
4498         break;
4499     }
4500
4501     case ResolveGlobal: {
4502         GPRTemporary globalObject(this);
4503         GPRTemporary resolveInfo(this);
4504         GPRTemporary resultTag(this);
4505         GPRTemporary resultPayload(this);
4506
4507         GPRReg globalObjectGPR = globalObject.gpr();
4508         GPRReg resolveInfoGPR = resolveInfo.gpr();
4509         GPRReg resultTagGPR = resultTag.gpr();
4510         GPRReg resultPayloadGPR = resultPayload.gpr();
4511
4512         ResolveGlobalData& data = m_jit.graph().m_resolveGlobalData[node->resolveGlobalDataIndex()];
4513         ResolveOperation* resolveOperationAddress = &(data.resolveOperations->data()[data.resolvePropertyIndex]);
4514
4515         // Check Structure of global object
4516         m_jit.move(JITCompiler::TrustedImmPtr(m_jit.globalObjectFor(node->codeOrigin)), globalObjectGPR);
4517         m_jit.move(JITCompiler::TrustedImmPtr(resolveOperationAddress), resolveInfoGPR);
4518         m_jit.loadPtr(JITCompiler::Address(resolveInfoGPR, OBJECT_OFFSETOF(ResolveOperation, m_structure)), resultPayloadGPR);
4519
4520         JITCompiler::Jump structuresNotMatch = m_jit.branchPtr(JITCompiler::NotEqual, resultPayloadGPR, JITCompiler::Address(globalObjectGPR, JSCell::structureOffset()));
4521
4522         // Fast case
4523         m_jit.loadPtr(JITCompiler::Address(globalObjectGPR, JSObject::butterflyOffset()), resultPayloadGPR);
4524         m_jit.load32(JITCompiler::Address(resolveInfoGPR, OBJECT_OFFSETOF(ResolveOperation, m_offset)), resolveInfoGPR);
4525 #if DFG_ENABLE(JIT_ASSERT)
4526         JITCompiler::Jump isOutOfLine = m_jit.branch32(JITCompiler::GreaterThanOrEqual, resolveInfoGPR, TrustedImm32(firstOutOfLineOffset));
4527         m_jit.breakpoint();
4528         isOutOfLine.link(&m_jit);
4529 #endif
4530         m_jit.neg32(resolveInfoGPR);
4531         m_jit.signExtend32ToPtr(resolveInfoGPR, resolveInfoGPR);
4532         m_jit.load32(JITCompiler::BaseIndex(resultPayloadGPR, resolveInfoGPR, JITCompiler::TimesEight, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag) + (firstOutOfLineOffset - 2) * static_cast<ptrdiff_t>(sizeof(JSValue))), resultTagGPR);
4533         m_jit.load32(JITCompiler::BaseIndex(resultPayloadGPR, resolveInfoGPR, JITCompiler::TimesEight, OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload) + (firstOutOfLineOffset - 2) * static_cast<ptrdiff_t>(sizeof(JSValue))), resultPayloadGPR);
4534
4535         addSlowPathGenerator(
4536             slowPathCall(
4537                 structuresNotMatch, this, operationResolveGlobal,
4538                 JSValueRegs(resultTagGPR, resultPayloadGPR), resolveInfoGPR, globalObjectGPR,
4539                 &m_jit.codeBlock()->identifier(data.identifierNumber)));
4540
4541         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4542         break;
4543     }
4544
4545     case CreateActivation: {
4546         JSValueOperand value(this, node->child1());
4547         GPRTemporary result(this, value, false);
4548
4549         GPRReg valueTagGPR = value.tagGPR();
4550         GPRReg valuePayloadGPR = value.payloadGPR();
4551         GPRReg resultGPR = result.gpr();
4552
4553         m_jit.move(valuePayloadGPR, resultGPR);
4554
4555         JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
4556
4557         addSlowPathGenerator(
4558             slowPathCall(notCreated, this, operationCreateActivation, resultGPR));
4559
4560         cellResult(resultGPR, node);
4561         break;
4562     }
4563
4564     case CreateArguments: {
4565         JSValueOperand value(this, node->child1());
4566         GPRTemporary result(this, value, false);
4567
4568         GPRReg valueTagGPR = value.tagGPR();
4569         GPRReg valuePayloadGPR = value.payloadGPR();
4570         GPRReg resultGPR = result.gpr();
4571
4572         m_jit.move(valuePayloadGPR, resultGPR);
4573
4574         JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
4575
4576         if (node->codeOrigin.inlineCallFrame) {
4577             addSlowPathGenerator(
4578                 slowPathCall(
4579                     notCreated, this, operationCreateInlinedArguments, resultGPR,
4580                     node->codeOrigin.inlineCallFrame));
4581         } else {
4582             addSlowPathGenerator(
4583                 slowPathCall(notCreated, this, operationCreateArguments, resultGPR));
4584         }
4585
4586         cellResult(resultGPR, node);
4587         break;
4588     }
4589
4590     case TearOffActivation: {
4591         JSValueOperand activationValue(this, node->child1());
4592         GPRTemporary scratch(this);
4593
4594         GPRReg activationValueTagGPR = activationValue.tagGPR();
4595         GPRReg activationValuePayloadGPR = activationValue.payloadGPR();
4596         GPRReg scratchGPR = scratch.gpr();
4597
4598         JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, activationValueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
4599
4600         SharedSymbolTable* symbolTable = m_jit.symbolTableFor(node->codeOrigin);
4601         int registersOffset = JSActivation::registersOffset(symbolTable);
4602
4603         int captureEnd = symbolTable->captureEnd();
4604         for (int i = symbolTable->captureStart(); i < captureEnd; ++i) {
4605             m_jit.loadPtr(
4606                 JITCompiler::Address(
4607                     GPRInfo::callFrameRegister, i * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4608                 scratchGPR);
4609             m_jit.storePtr(
4610                 scratchGPR, JITCompiler::Address(
4611                     activationValuePayloadGPR, registersOffset + i * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
4612             m_jit.loadPtr(
4613                 JITCompiler::Address(
4614                     GPRInfo::callFrameRegister, i * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4615                 scratchGPR);
4616             m_jit.storePtr(
4617                 scratchGPR, JITCompiler::Address(
4618                     activationValuePayloadGPR, registersOffset + i * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
4619         }
4620         m_jit.addPtr(TrustedImm32(registersOffset), activationValuePayloadGPR, scratchGPR);
4621         m_jit.storePtr(scratchGPR, JITCompiler::Address(activationValuePayloadGPR, JSActivation::offsetOfRegisters()));
4622
4623         notCreated.link(&m_jit);
4624         noResult(node);
4625         break;
4626     }
4627
4628     case TearOffArguments: {
4629         JSValueOperand unmodifiedArgumentsValue(this, node->child1());
4630         JSValueOperand activationValue(this, node->child2());
4631         GPRReg unmodifiedArgumentsValuePayloadGPR = unmodifiedArgumentsValue.payloadGPR();
4632         GPRReg activationValuePayloadGPR = activationValue.payloadGPR();
4633
4634         JITCompiler::Jump created = m_jit.branchTest32(
4635             JITCompiler::NonZero, unmodifiedArgumentsValuePayloadGPR);
4636
4637         if (node->codeOrigin.inlineCallFrame) {
4638             addSlowPathGenerator(
4639                 slowPathCall(
4640                     created, this, operationTearOffInlinedArguments, NoResult,
4641                     unmodifiedArgumentsValuePayloadGPR, activationValuePayloadGPR, node->codeOrigin.inlineCallFrame));
4642         } else {
4643             addSlowPathGenerator(
4644                 slowPathCall(
4645                     created, this, operationTearOffArguments, NoResult,
4646                     unmodifiedArgumentsValuePayloadGPR, activationValuePayloadGPR));
4647         }
4648
4649         noResult(node);
4650         break;
4651     }
4652
4653     case CheckArgumentsNotCreated: {
4654         ASSERT(!isEmptySpeculation(
4655             m_state.variables().operand(
4656                 m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type));
4657         speculationCheck(
4658             Uncountable, JSValueRegs(), 0,
4659             m_jit.branch32(
4660                 JITCompiler::NotEqual,
4661                 JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4662                 TrustedImm32(JSValue::EmptyValueTag)));
4663         noResult(node);
4664         break;
4665     }
4666
4667     case GetMyArgumentsLength: {
4668         GPRTemporary result(this);
4669         GPRReg resultGPR = result.gpr();
4670
4671         if (!isEmptySpeculation(
4672                 m_state.variables().operand(
4673                     m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) {
4674             speculationCheck(
4675                 ArgumentsEscaped, JSValueRegs(), 0,
4676                 m_jit.branch32(
4677                     JITCompiler::NotEqual,
4678                     JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4679                     TrustedImm32(JSValue::EmptyValueTag)));
4680         }
4681
4682         ASSERT(!node->codeOrigin.inlineCallFrame);
4683         m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR);
4684         m_jit.sub32(TrustedImm32(1), resultGPR);
4685         integerResult(resultGPR, node);
4686         break;
4687     }
4688
4689     case GetMyArgumentsLengthSafe: {
4690         GPRTemporary resultPayload(this);
4691         GPRTemporary resultTag(this);
4692         GPRReg resultPayloadGPR = resultPayload.gpr();
4693         GPRReg resultTagGPR = resultTag.gpr();
4694
4695         JITCompiler::Jump created = m_jit.branch32(
4696             JITCompiler::NotEqual,
4697             JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4698             TrustedImm32(JSValue::EmptyValueTag));
4699
4700         if (node->codeOrigin.inlineCallFrame) {
4701             m_jit.move(
4702                 Imm32(node->codeOrigin.inlineCallFrame->arguments.size() - 1),
4703                 resultPayloadGPR);
4704         } else {
4705             m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultPayloadGPR);
4706             m_jit.sub32(TrustedImm32(1), resultPayloadGPR);
4707         }
4708         m_jit.move(TrustedImm32(JSValue::Int32Tag), resultTagGPR);
4709
4710         // FIXME: the slow path generator should perform a forward speculation that the
4711         // result is an integer. For now we postpone the speculation by having this return
4712         // a JSValue.
4713
4714         addSlowPathGenerator(
4715             slowPathCall(
4716                 created, this, operationGetArgumentsLength,
4717                 JSValueRegs(resultTagGPR, resultPayloadGPR),
4718                 m_jit.argumentsRegisterFor(node->codeOrigin)));
4719
4720         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4721         break;
4722     }
4723
4724     case GetMyArgumentByVal: {
4725         SpeculateStrictInt32Operand index(this, node->child1());
4726         GPRTemporary resultPayload(this);
4727         GPRTemporary resultTag(this);
4728         GPRReg indexGPR = index.gpr();
4729         GPRReg resultPayloadGPR = resultPayload.gpr();
4730         GPRReg resultTagGPR = resultTag.gpr();
4731
4732         if (!isEmptySpeculation(
4733                 m_state.variables().operand(
4734                     m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) {
4735             speculationCheck(
4736                 ArgumentsEscaped, JSValueRegs(), 0,
4737                 m_jit.branch32(
4738                     JITCompiler::NotEqual,
4739                     JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4740                     TrustedImm32(JSValue::EmptyValueTag)));
4741         }
4742
4743         m_jit.add32(TrustedImm32(1), indexGPR, resultPayloadGPR);
4744
4745         if (node->codeOrigin.inlineCallFrame) {
4746             speculationCheck(
4747                 Uncountable, JSValueRegs(), 0,
4748                 m_jit.branch32(
4749                     JITCompiler::AboveOrEqual,
4750                     resultPayloadGPR,
4751                     Imm32(node->codeOrigin.inlineCallFrame->arguments.size())));
4752         } else {
4753             speculationCheck(
4754                 Uncountable, JSValueRegs(), 0,
4755                 m_jit.branch32(
4756                     JITCompiler::AboveOrEqual,
4757                     resultPayloadGPR,
4758                     JITCompiler::payloadFor(JSStack::ArgumentCount)));
4759         }
4760
4761         JITCompiler::JumpList slowArgument;
4762         JITCompiler::JumpList slowArgumentOutOfBounds;
4763         if (const SlowArgument* slowArguments = m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) {
4764             slowArgumentOutOfBounds.append(
4765                 m_jit.branch32(
4766                     JITCompiler::AboveOrEqual, indexGPR,
4767                     Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount())));
4768
4769             COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes);
4770             m_jit.move(ImmPtr(slowArguments), resultPayloadGPR);
4771             m_jit.load32(
4772                 JITCompiler::BaseIndex(
4773                     resultPayloadGPR, indexGPR, JITCompiler::TimesEight,
4774                     OBJECT_OFFSETOF(SlowArgument, index)),
4775                 resultPayloadGPR);
4776
4777             m_jit.load32(
4778                 JITCompiler::BaseIndex(
4779                     GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4780                     m_jit.offsetOfLocals(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4781                 resultTagGPR);
4782             m_jit.load32(
4783                 JITCompiler::BaseIndex(
4784                     GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4785                     m_jit.offsetOfLocals(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4786                 resultPayloadGPR);
4787             slowArgument.append(m_jit.jump());
4788         }
4789         slowArgumentOutOfBounds.link(&m_jit);
4790
4791         m_jit.neg32(resultPayloadGPR);
4792
4793         m_jit.load32(
4794             JITCompiler::BaseIndex(
4795                 GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4796                 m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4797             resultTagGPR);
4798         m_jit.load32(
4799             JITCompiler::BaseIndex(
4800                 GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4801                 m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4802             resultPayloadGPR);
4803
4804         slowArgument.link(&m_jit);
4805         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4806         break;
4807     }
4808     case GetMyArgumentByValSafe: {
4809         SpeculateStrictInt32Operand index(this, node->child1());
4810         GPRTemporary resultPayload(this);
4811         GPRTemporary resultTag(this);
4812         GPRReg indexGPR = index.gpr();
4813         GPRReg resultPayloadGPR = resultPayload.gpr();
4814         GPRReg resultTagGPR = resultTag.gpr();
4815
4816         JITCompiler::JumpList slowPath;
4817         slowPath.append(
4818             m_jit.branch32(
4819                 JITCompiler::NotEqual,
4820                 JITCompiler::tagFor(m_jit.argumentsRegisterFor(node->codeOrigin)),
4821                 TrustedImm32(JSValue::EmptyValueTag)));
4822
4823         m_jit.add32(TrustedImm32(1), indexGPR, resultPayloadGPR);
4824         if (node->codeOrigin.inlineCallFrame) {
4825             slowPath.append(
4826                 m_jit.branch32(
4827                     JITCompiler::AboveOrEqual,
4828                     resultPayloadGPR,
4829                     Imm32(node->codeOrigin.inlineCallFrame->arguments.size())));
4830         } else {
4831             slowPath.append(
4832                 m_jit.branch32(
4833                     JITCompiler::AboveOrEqual,
4834                     resultPayloadGPR,
4835                     JITCompiler::payloadFor(JSStack::ArgumentCount)));
4836         }
4837
4838         JITCompiler::JumpList slowArgument;
4839         JITCompiler::JumpList slowArgumentOutOfBounds;
4840         if (const SlowArgument* slowArguments = m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) {
4841             slowArgumentOutOfBounds.append(
4842                 m_jit.branch32(
4843                     JITCompiler::AboveOrEqual, indexGPR,
4844                     Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount())));
4845
4846             COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes);
4847             m_jit.move(ImmPtr(slowArguments), resultPayloadGPR);
4848             m_jit.load32(
4849                 JITCompiler::BaseIndex(
4850                     resultPayloadGPR, indexGPR, JITCompiler::TimesEight,
4851                     OBJECT_OFFSETOF(SlowArgument, index)),
4852                 resultPayloadGPR);
4853             m_jit.load32(
4854                 JITCompiler::BaseIndex(
4855                     GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4856                     m_jit.offsetOfLocals(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4857                 resultTagGPR);
4858             m_jit.load32(
4859                 JITCompiler::BaseIndex(
4860                     GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4861                     m_jit.offsetOfLocals(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4862                 resultPayloadGPR);
4863             slowArgument.append(m_jit.jump());
4864         }
4865         slowArgumentOutOfBounds.link(&m_jit);
4866
4867         m_jit.neg32(resultPayloadGPR);
4868
4869         m_jit.load32(
4870             JITCompiler::BaseIndex(
4871                 GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4872                 m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
4873             resultTagGPR);
4874         m_jit.load32(
4875             JITCompiler::BaseIndex(
4876                 GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight,
4877                 m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
4878             resultPayloadGPR);
4879
4880         if (node->codeOrigin.inlineCallFrame) {
4881             addSlowPathGenerator(
4882                 slowPathCall(
4883                     slowPath, this, operationGetInlinedArgumentByVal,
4884                     JSValueRegs(resultTagGPR, resultPayloadGPR),
4885                     m_jit.argumentsRegisterFor(node->codeOrigin),
4886                     node->codeOrigin.inlineCallFrame, indexGPR));
4887         } else {
4888             addSlowPathGenerator(
4889                 slowPathCall(
4890                     slowPath, this, operationGetArgumentByVal,
4891                     JSValueRegs(resultTagGPR, resultPayloadGPR),
4892                     m_jit.argumentsRegisterFor(node->codeOrigin), indexGPR));
4893         }
4894
4895         slowArgument.link(&m_jit);
4896         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4897         break;
4898     }
4899
4900     case NewFunctionNoCheck:
4901         compileNewFunctionNoCheck(node);
4902         break;
4903
4904     case NewFunction: {
4905         JSValueOperand value(this, node->child1());
4906         GPRTemporary resultTag(this, value);
4907         GPRTemporary resultPayload(this, value, false);
4908
4909         GPRReg valueTagGPR = value.tagGPR();
4910         GPRReg valuePayloadGPR = value.payloadGPR();
4911         GPRReg resultTagGPR = resultTag.gpr();
4912         GPRReg resultPayloadGPR = resultPayload.gpr();
4913
4914         m_jit.move(valuePayloadGPR, resultPayloadGPR);
4915         m_jit.move(valueTagGPR, resultTagGPR);
4916
4917         JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag));
4918
4919         addSlowPathGenerator(
4920             slowPathCall(
4921                 notCreated, this, operationNewFunction, JSValueRegs(resultTagGPR, resultPayloadGPR),
4922                 m_jit.codeBlock()->functionDecl(node->functionDeclIndex())));
4923
4924         jsValueResult(resultTagGPR, resultPayloadGPR, node);
4925         break;
4926     }
4927
4928     case NewFunctionExpression:
4929         compileNewFunctionExpression(node);
4930         break;
4931
4932     case GarbageValue:
4933         // We should never get to the point of code emission for a GarbageValue
4934         CRASH();
4935         break;
4936
4937     case ForceOSRExit: {
4938         terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0);
4939         break;
4940     }
4941
4942     case CheckWatchdogTimer:
4943         speculationCheck(
4944             WatchdogTimerFired, JSValueRegs(), 0,
4945             m_jit.branchTest8(
4946                 JITCompiler::NonZero,
4947                 JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog.timerDidFireAddress())));
4948         break;
4949
4950     case CountExecution:
4951         m_jit.add64(TrustedImm32(1), MacroAssembler::AbsoluteAddress(node->executionCounter()->address()));
4952         break;
4953
4954     case Phantom:
4955         DFG_NODE_DO_TO_CHILDREN(m_jit.graph(), node, speculate);
4956         noResult(node);
4957         break;
4958
4959     case PhantomLocal:
4960         // This is a no-op.
4961         noResult(node);
4962         break;
4963
4964     case Unreachable:
4965         RELEASE_ASSERT_NOT_REACHED();
4966         break;
4967
4968     case Nop:
4969     case LastNodeType:
4970         RELEASE_ASSERT_NOT_REACHED();
4971         break;
4972     }
4973
4974 #if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION)
4975     m_jit.clearRegisterAllocationOffsets();
4976 #endif
4977
4978     if (!m_compileOkay)
4979         return;
4980
4981     if (node->hasResult() && node->mustGenerate())
4982         use(node);
4983 }
4984
4985 #endif
4986
4987 } } // namespace JSC::DFG
4988
4989 #endif