X-Git-Url: https://git.saurik.com/apple/javascriptcore.git/blobdiff_plain/93a3786624b2768d89bfa27e46598dc64e2fb70a..ed1e77d3adeb83d26fd1dfb16dd84cabdcefd250:/dfg/DFGSpeculativeJIT64.cpp diff --git a/dfg/DFGSpeculativeJIT64.cpp b/dfg/DFGSpeculativeJIT64.cpp index bf5361e..2c78f9a 100644 --- a/dfg/DFGSpeculativeJIT64.cpp +++ b/dfg/DFGSpeculativeJIT64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, 2012, 2013 Apple Inc. All rights reserved. + * Copyright (C) 2011-2015 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -28,148 +28,92 @@ #if ENABLE(DFG_JIT) -#include "Arguments.h" #include "ArrayPrototype.h" +#include "DFGAbstractInterpreterInlines.h" #include "DFGCallArrayAllocatorSlowPathGenerator.h" +#include "DFGOperations.h" #include "DFGSlowPathGenerator.h" -#include "JSCJSValueInlines.h" +#include "Debugger.h" +#include "DirectArguments.h" +#include "GetterSetter.h" +#include "JSCInlines.h" +#include "JSEnvironmentRecord.h" +#include "JSLexicalEnvironment.h" +#include "JSPropertyNameEnumerator.h" #include "ObjectPrototype.h" +#include "SetupVarargsFrame.h" +#include "SpillRegistersMode.h" +#include "TypeProfilerLog.h" namespace JSC { namespace DFG { #if USE(JSVALUE64) -GPRReg SpeculativeJIT::fillInteger(Edge edge, DataFormat& returnFormat) +void SpeculativeJIT::boxInt52(GPRReg sourceGPR, GPRReg targetGPR, DataFormat format) { - ASSERT(!needsTypeCheck(edge, SpecInt32)); + GPRReg tempGPR; + if (sourceGPR == targetGPR) + tempGPR = allocate(); + else + tempGPR = targetGPR; - VirtualRegister virtualRegister = edge->virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; - - if (info.registerFormat() == DataFormatNone) { - GPRReg gpr = allocate(); - - if (edge->hasConstant()) { - m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - if (isInt32Constant(edge.node())) { - m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr); - info.fillInteger(*m_stream, gpr); - returnFormat = DataFormatInteger; - return gpr; - } - if (isNumberConstant(edge.node())) { - JSValue jsValue = jsNumber(valueOfNumberConstant(edge.node())); - m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr); - } else { - ASSERT(isJSConstant(edge.node())); - JSValue jsValue = valueOfJSConstant(edge.node()); - m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); - } - } else if (info.spillFormat() == DataFormatInteger) { - m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr); - // Tag it, since fillInteger() is used when we want a boxed integer. - m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr); - } else { - RELEASE_ASSERT(info.spillFormat() == DataFormatJS || info.spillFormat() == DataFormatJSInteger); - m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - } + FPRReg fpr = fprAllocate(); - // Since we statically know that we're filling an integer, and values - // in the JSStack are boxed, this must be DataFormatJSInteger. - // We will check this with a jitAssert below. - info.fillJSValue(*m_stream, gpr, DataFormatJSInteger); - unlock(gpr); - } + if (format == DataFormatInt52) + m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), sourceGPR); + else + ASSERT(format == DataFormatStrictInt52); - switch (info.registerFormat()) { - case DataFormatNone: - // Should have filled, above. - case DataFormatJSDouble: - case DataFormatDouble: - case DataFormatJS: - case DataFormatCell: - case DataFormatJSCell: - case DataFormatBoolean: - case DataFormatJSBoolean: - case DataFormatStorage: - // Should only be calling this function if we know this operand to be integer. - RELEASE_ASSERT_NOT_REACHED(); - - case DataFormatJSInteger: { - GPRReg gpr = info.gpr(); - m_gprs.lock(gpr); - m_jit.jitAssertIsJSInt32(gpr); - returnFormat = DataFormatJSInteger; - return gpr; - } - - case DataFormatInteger: { - GPRReg gpr = info.gpr(); - m_gprs.lock(gpr); - m_jit.jitAssertIsInt32(gpr); - returnFormat = DataFormatInteger; - return gpr; - } - - default: - RELEASE_ASSERT_NOT_REACHED(); - return InvalidGPRReg; - } + m_jit.boxInt52(sourceGPR, targetGPR, tempGPR, fpr); + + if (format == DataFormatInt52 && sourceGPR != targetGPR) + m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), sourceGPR); + + if (tempGPR != targetGPR) + unlock(tempGPR); + + unlock(fpr); } GPRReg SpeculativeJIT::fillJSValue(Edge edge) { VirtualRegister virtualRegister = edge->virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; + GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { GPRReg gpr = allocate(); if (edge->hasConstant()) { - if (isInt32Constant(edge.node())) { - info.fillJSValue(*m_stream, gpr, DataFormatJSInteger); - JSValue jsValue = jsNumber(valueOfInt32Constant(edge.node())); - m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr); - } else if (isNumberConstant(edge.node())) { - info.fillJSValue(*m_stream, gpr, DataFormatJSDouble); - JSValue jsValue(JSValue::EncodeAsDouble, valueOfNumberConstant(edge.node())); - m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr); - } else { - ASSERT(isJSConstant(edge.node())); - JSValue jsValue = valueOfJSConstant(edge.node()); - m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJS); - } - + JSValue jsValue = edge->asJSValue(); + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); + info.fillJSValue(*m_stream, gpr, DataFormatJS); m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); } else { DataFormat spillFormat = info.spillFormat(); m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - if (spillFormat == DataFormatInteger) { + switch (spillFormat) { + case DataFormatInt32: { m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr); m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr); - spillFormat = DataFormatJSInteger; - } else { + spillFormat = DataFormatJSInt32; + break; + } + + default: m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - if (spillFormat == DataFormatDouble) { - // Need to box the double, since we want a JSValue. - m_jit.sub64(GPRInfo::tagTypeNumberRegister, gpr); - spillFormat = DataFormatJSDouble; - } else - RELEASE_ASSERT(spillFormat & DataFormatJS); + DFG_ASSERT(m_jit.graph(), m_currentNode, spillFormat & DataFormatJS); + break; } info.fillJSValue(*m_stream, gpr, spillFormat); } return gpr; } - case DataFormatInteger: { + case DataFormatInt32: { GPRReg gpr = info.gpr(); // If the register has already been locked we need to take a copy. - // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInteger, not DataFormatJSInteger. + // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInt32, not DataFormatJSInt32. if (m_gprs.isLocked(gpr)) { GPRReg result = allocate(); m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr, result); @@ -177,26 +121,14 @@ GPRReg SpeculativeJIT::fillJSValue(Edge edge) } m_gprs.lock(gpr); m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJSInteger); - return gpr; - } - - case DataFormatDouble: { - FPRReg fpr = info.fpr(); - GPRReg gpr = boxDouble(fpr); - - // Update all info - info.fillJSValue(*m_stream, gpr, DataFormatJSDouble); - m_fprs.release(fpr); - m_gprs.retain(gpr, virtualRegister, SpillOrderJS); - + info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); return gpr; } case DataFormatCell: // No retag required on JSVALUE64! case DataFormatJS: - case DataFormatJSInteger: + case DataFormatJSInt32: case DataFormatJSDouble: case DataFormatJSCell: case DataFormatJSBoolean: { @@ -207,121 +139,56 @@ GPRReg SpeculativeJIT::fillJSValue(Edge edge) case DataFormatBoolean: case DataFormatStorage: + case DataFormatDouble: + case DataFormatInt52: // this type currently never occurs - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Corrupt data format"); return InvalidGPRReg; } } -void SpeculativeJIT::nonSpeculativeUInt32ToNumber(Node* node) +void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg resultGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode) { - IntegerOperand op1(this, node->child1()); - FPRTemporary boxer(this); - GPRTemporary result(this, op1); - - JITCompiler::Jump positive = m_jit.branch32(MacroAssembler::GreaterThanOrEqual, op1.gpr(), TrustedImm32(0)); - - m_jit.convertInt32ToDouble(op1.gpr(), boxer.fpr()); - m_jit.addDouble(JITCompiler::AbsoluteAddress(&AssemblyHelpers::twoToThe32), boxer.fpr()); - - boxDouble(boxer.fpr(), result.gpr()); - - JITCompiler::Jump done = m_jit.jump(); - - positive.link(&m_jit); + JITGetByIdGenerator gen( + m_jit.codeBlock(), codeOrigin, usedRegisters(), JSValueRegs(baseGPR), + JSValueRegs(resultGPR), spillMode); + gen.generateFastPath(m_jit); - m_jit.or64(GPRInfo::tagTypeNumberRegister, op1.gpr(), result.gpr()); + JITCompiler::JumpList slowCases; + if (slowPathTarget.isSet()) + slowCases.append(slowPathTarget); + slowCases.append(gen.slowPathJump()); - done.link(&m_jit); + auto slowPath = slowPathCall( + slowCases, this, operationGetByIdOptimize, resultGPR, gen.stubInfo(), baseGPR, + identifierUID(identifierNumber), spillMode); - jsValueResult(result.gpr(), m_currentNode); + m_jit.addGetById(gen, slowPath.get()); + addSlowPathGenerator(WTF::move(slowPath)); } -void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg resultGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode) +void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode) { - JITCompiler::DataLabelPtr structureToCompare; - JITCompiler::PatchableJump structureCheck = m_jit.patchableBranchPtrWithPatch(JITCompiler::NotEqual, JITCompiler::Address(baseGPR, JSCell::structureOffset()), structureToCompare, JITCompiler::TrustedImmPtr(reinterpret_cast(unusedPointer))); - - JITCompiler::ConvertibleLoadLabel propertyStorageLoad = - m_jit.convertibleLoadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), resultGPR); - JITCompiler::DataLabelCompact loadWithPatch = m_jit.load64WithCompactAddressOffsetPatch(JITCompiler::Address(resultGPR, 0), resultGPR); - - JITCompiler::Label doneLabel = m_jit.label(); + JITPutByIdGenerator gen( + m_jit.codeBlock(), codeOrigin, usedRegisters(), JSValueRegs(baseGPR), + JSValueRegs(valueGPR), scratchGPR, spillMode, m_jit.ecmaModeFor(codeOrigin), putKind); - OwnPtr slowPath; - if (!slowPathTarget.isSet()) { - slowPath = slowPathCall( - structureCheck.m_jump, this, operationGetByIdOptimize, resultGPR, baseGPR, - identifier(identifierNumber), spillMode); - } else { - JITCompiler::JumpList slowCases; - slowCases.append(structureCheck.m_jump); + gen.generateFastPath(m_jit); + + JITCompiler::JumpList slowCases; + if (slowPathTarget.isSet()) slowCases.append(slowPathTarget); - slowPath = slowPathCall( - slowCases, this, operationGetByIdOptimize, resultGPR, baseGPR, - identifier(identifierNumber), spillMode); - } - m_jit.addPropertyAccess( - PropertyAccessRecord( - codeOrigin, structureToCompare, structureCheck, propertyStorageLoad, loadWithPatch, - slowPath.get(), doneLabel, safeCast(baseGPR), safeCast(resultGPR), - usedRegisters(), - spillMode == NeedToSpill ? PropertyAccessRecord::RegistersInUse : PropertyAccessRecord::RegistersFlushed)); - addSlowPathGenerator(slowPath.release()); -} - -void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg valueGPR, Edge valueUse, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget) -{ + slowCases.append(gen.slowPathJump()); - JITCompiler::DataLabelPtr structureToCompare; - JITCompiler::PatchableJump structureCheck = m_jit.patchableBranchPtrWithPatch(JITCompiler::NotEqual, JITCompiler::Address(baseGPR, JSCell::structureOffset()), structureToCompare, JITCompiler::TrustedImmPtr(reinterpret_cast(unusedPointer))); + auto slowPath = slowPathCall( + slowCases, this, gen.slowPathFunction(), NoResult, gen.stubInfo(), valueGPR, baseGPR, + identifierUID(identifierNumber)); - writeBarrier(baseGPR, valueGPR, valueUse, WriteBarrierForPropertyAccess, scratchGPR); - - JITCompiler::ConvertibleLoadLabel propertyStorageLoad = - m_jit.convertibleLoadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR); - JITCompiler::DataLabel32 storeWithPatch = m_jit.store64WithAddressOffsetPatch(valueGPR, JITCompiler::Address(scratchGPR, 0)); - - JITCompiler::Label doneLabel = m_jit.label(); - - V_DFGOperation_EJCI optimizedCall; - if (m_jit.strictModeFor(m_currentNode->codeOrigin)) { - if (putKind == Direct) - optimizedCall = operationPutByIdDirectStrictOptimize; - else - optimizedCall = operationPutByIdStrictOptimize; - } else { - if (putKind == Direct) - optimizedCall = operationPutByIdDirectNonStrictOptimize; - else - optimizedCall = operationPutByIdNonStrictOptimize; - } - OwnPtr slowPath; - if (!slowPathTarget.isSet()) { - slowPath = slowPathCall( - structureCheck.m_jump, this, optimizedCall, NoResult, valueGPR, baseGPR, - identifier(identifierNumber)); - } else { - JITCompiler::JumpList slowCases; - slowCases.append(structureCheck.m_jump); - slowCases.append(slowPathTarget); - slowPath = slowPathCall( - slowCases, this, optimizedCall, NoResult, valueGPR, baseGPR, - identifier(identifierNumber)); - } - RegisterSet currentlyUsedRegisters = usedRegisters(); - currentlyUsedRegisters.clear(scratchGPR); - ASSERT(currentlyUsedRegisters.get(baseGPR)); - ASSERT(currentlyUsedRegisters.get(valueGPR)); - m_jit.addPropertyAccess( - PropertyAccessRecord( - codeOrigin, structureToCompare, structureCheck, propertyStorageLoad, - JITCompiler::DataLabelCompact(storeWithPatch.label()), slowPath.get(), doneLabel, - safeCast(baseGPR), safeCast(valueGPR), currentlyUsedRegisters)); - addSlowPathGenerator(slowPath.release()); + m_jit.addPutById(gen, slowPath.get()); + addSlowPathGenerator(WTF::move(slowPath)); } void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool invert) @@ -329,28 +196,30 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool inv JSValueOperand arg(this, operand); GPRReg argGPR = arg.gpr(); - GPRTemporary result(this, arg); + GPRTemporary result(this, Reuse, arg); GPRReg resultGPR = result.gpr(); JITCompiler::Jump notCell; JITCompiler::Jump notMasqueradesAsUndefined; - if (m_jit.graph().globalObjectFor(operand->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) { + if (masqueradesAsUndefinedWatchpointIsStillValid()) { if (!isKnownCell(operand.node())) - notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister); + notCell = m_jit.branchIfNotCell(JSValueRegs(argGPR)); - m_jit.graph().globalObjectFor(operand->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultGPR); notMasqueradesAsUndefined = m_jit.jump(); } else { GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); + GPRTemporary scratch(this); if (!isKnownCell(operand.node())) - notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister); - - m_jit.loadPtr(JITCompiler::Address(argGPR, JSCell::structureOffset()), resultGPR); - JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined)); + notCell = m_jit.branchIfNotCell(JSValueRegs(argGPR)); + + JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::NonZero, + JITCompiler::Address(argGPR, JSCell::typeInfoFlagsOffset()), + JITCompiler::TrustedImm32(MasqueradesAsUndefined)); m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultGPR); notMasqueradesAsUndefined = m_jit.jump(); @@ -358,7 +227,8 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool inv isMasqueradesAsUndefined.link(&m_jit); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(JITCompiler::TrustedImmPtr(m_jit.graph().globalObjectFor(operand->codeOrigin)), localGlobalObjectGPR); + m_jit.move(JITCompiler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)), localGlobalObjectGPR); + m_jit.emitLoadStructure(argGPR, resultGPR, scratch.gpr()); m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR); m_jit.comparePtr(invert ? JITCompiler::NotEqual : JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultGPR); } @@ -383,12 +253,12 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool inv void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branchNode, bool invert) { - BlockIndex taken = branchNode->takenBlockIndex(); - BlockIndex notTaken = branchNode->notTakenBlockIndex(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; if (taken == nextBlock()) { invert = !invert; - BlockIndex tmp = taken; + BasicBlock* tmp = taken; taken = notTaken; notTaken = tmp; } @@ -396,30 +266,33 @@ void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branch JSValueOperand arg(this, operand); GPRReg argGPR = arg.gpr(); - GPRTemporary result(this, arg); + GPRTemporary result(this, Reuse, arg); GPRReg resultGPR = result.gpr(); JITCompiler::Jump notCell; - if (m_jit.graph().globalObjectFor(operand->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) { + if (masqueradesAsUndefinedWatchpointIsStillValid()) { if (!isKnownCell(operand.node())) - notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister); - - m_jit.graph().globalObjectFor(operand->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); + notCell = m_jit.branchIfNotCell(JSValueRegs(argGPR)); + jump(invert ? taken : notTaken, ForceJump); } else { GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); + GPRTemporary scratch(this); if (!isKnownCell(operand.node())) - notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister); - - m_jit.loadPtr(JITCompiler::Address(argGPR, JSCell::structureOffset()), resultGPR); - branchTest8(JITCompiler::Zero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined), invert ? taken : notTaken); + notCell = m_jit.branchIfNotCell(JSValueRegs(argGPR)); + + branchTest8(JITCompiler::Zero, + JITCompiler::Address(argGPR, JSCell::typeInfoFlagsOffset()), + JITCompiler::TrustedImm32(MasqueradesAsUndefined), + invert ? taken : notTaken); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(TrustedImmPtr(m_jit.graph().globalObjectFor(operand->codeOrigin)), localGlobalObjectGPR); + m_jit.move(TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)), localGlobalObjectGPR); + m_jit.emitLoadStructure(argGPR, resultGPR, scratch.gpr()); m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR); branchPtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, invert ? notTaken : taken); } @@ -441,9 +314,9 @@ bool SpeculativeJIT::nonSpeculativeCompareNull(Node* node, Edge operand, bool in { unsigned branchIndexInBlock = detectPeepHoleBranch(); if (branchIndexInBlock != UINT_MAX) { - Node* branchNode = m_jit.graph().m_blocks[m_block]->at(branchIndexInBlock); + Node* branchNode = m_block->at(branchIndexInBlock); - RELEASE_ASSERT(node->adjustedRefCount() == 1); + DFG_ASSERT(m_jit.graph(), node, node->adjustedRefCount() == 1); nonSpeculativePeepholeBranchNull(operand, branchNode, invert); @@ -460,10 +333,10 @@ bool SpeculativeJIT::nonSpeculativeCompareNull(Node* node, Edge operand, bool in return false; } -void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction) +void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction) { - BlockIndex taken = branchNode->takenBlockIndex(); - BlockIndex notTaken = branchNode->notTakenBlockIndex(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; JITCompiler::ResultCondition callResultCondition = JITCompiler::NonZero; @@ -472,7 +345,7 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, if (taken == nextBlock()) { cond = JITCompiler::invert(cond); callResultCondition = JITCompiler::Zero; - BlockIndex tmp = taken; + BasicBlock* tmp = taken; taken = notTaken; notTaken = tmp; } @@ -485,7 +358,7 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, JITCompiler::JumpList slowPath; if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) { - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); arg1.use(); @@ -496,7 +369,7 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, branchTest32(callResultCondition, resultGPR, taken); } else { - GPRTemporary result(this, arg2); + GPRTemporary result(this, Reuse, arg2); GPRReg resultGPR = result.gpr(); arg1.use(); @@ -524,18 +397,18 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, jump(notTaken); - m_indexInBlock = m_jit.graph().m_blocks[m_block]->size() - 1; + m_indexInBlock = m_block->size() - 1; m_currentNode = branchNode; } template class CompareAndBoxBooleanSlowPathGenerator - : public CallSlowPathGenerator { + : public CallSlowPathGenerator { public: CompareAndBoxBooleanSlowPathGenerator( JumpType from, SpeculativeJIT* jit, - S_DFGOperation_EJJ function, GPRReg result, GPRReg arg1, GPRReg arg2) - : CallSlowPathGenerator( + S_JITOperation_EJJ function, GPRReg result, GPRReg arg1, GPRReg arg2) + : CallSlowPathGenerator( from, jit, function, NeedToSpill, result) , m_arg1(arg1) , m_arg2(arg2) @@ -543,7 +416,7 @@ public: } protected: - virtual void generateInternal(SpeculativeJIT* jit) + virtual void generateInternal(SpeculativeJIT* jit) override { this->setUp(jit); this->recordCall(jit->callOperation(this->m_function, this->m_result, m_arg1, m_arg2)); @@ -557,8 +430,9 @@ private: GPRReg m_arg2; }; -void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_DFGOperation_EJJ helperFunction) +void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction) { + ASSERT(node->isBinaryUseKind(UntypedUse)); JSValueOperand arg1(this, node->child1()); JSValueOperand arg2(this, node->child2()); GPRReg arg1GPR = arg1.gpr(); @@ -567,7 +441,7 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler JITCompiler::JumpList slowPath; if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) { - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); arg1.use(); @@ -579,7 +453,7 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler m_jit.or32(TrustedImm32(ValueFalse), resultGPR); jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly); } else { - GPRTemporary result(this, arg2); + GPRTemporary result(this, Reuse, arg2); GPRReg resultGPR = result.gpr(); arg1.use(); @@ -594,9 +468,8 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler m_jit.or32(TrustedImm32(ValueFalse), resultGPR); if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) { - addSlowPathGenerator(adoptPtr( - new CompareAndBoxBooleanSlowPathGenerator( - slowPath, this, helperFunction, resultGPR, arg1GPR, arg2GPR))); + addSlowPathGenerator(std::make_unique>( + slowPath, this, helperFunction, resultGPR, arg1GPR, arg2GPR)); } jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly); @@ -605,14 +478,14 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode, bool invert) { - BlockIndex taken = branchNode->takenBlockIndex(); - BlockIndex notTaken = branchNode->notTakenBlockIndex(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; // The branch instruction will branch to the taken block. // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through. if (taken == nextBlock()) { invert = !invert; - BlockIndex tmp = taken; + BasicBlock* tmp = taken; taken = notTaken; notTaken = tmp; } @@ -726,11 +599,9 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) m_jit.move(JITCompiler::TrustedImm64(JSValue::encode(jsBoolean(!invert))), resultGPR); - addSlowPathGenerator( - adoptPtr( - new CompareAndBoxBooleanSlowPathGenerator( + addSlowPathGenerator(std::make_unique>( slowPathCases, this, operationCompareStrictEq, resultGPR, arg1GPR, - arg2GPR))); + arg2GPR)); done.link(&m_jit); } @@ -738,69 +609,182 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly); } -void SpeculativeJIT::emitCall(Node* node) +void SpeculativeJIT::compileMiscStrictEq(Node* node) { - if (node->op() != Call) - RELEASE_ASSERT(node->op() == Construct); - - // For constructors, the this argument is not passed but we have to make space - // for it. - int dummyThisArgument = node->op() == Call ? 0 : 1; - - CallLinkInfo::CallType callType = node->op() == Call ? CallLinkInfo::Call : CallLinkInfo::Construct; + JSValueOperand op1(this, node->child1(), ManualOperandSpeculation); + JSValueOperand op2(this, node->child2(), ManualOperandSpeculation); + GPRTemporary result(this); - Edge calleeEdge = m_jit.graph().m_varArgChildren[node->firstChild()]; - JSValueOperand callee(this, calleeEdge); - GPRReg calleeGPR = callee.gpr(); - use(calleeEdge); + if (node->child1().useKind() == MiscUse) + speculateMisc(node->child1(), op1.jsValueRegs()); + if (node->child2().useKind() == MiscUse) + speculateMisc(node->child2(), op2.jsValueRegs()); - // The call instruction's first child is the function; the subsequent children are the - // arguments. - int numPassedArgs = node->numChildren() - 1; + m_jit.compare64(JITCompiler::Equal, op1.gpr(), op2.gpr(), result.gpr()); + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), node, DataFormatJSBoolean); +} + +void SpeculativeJIT::emitCall(Node* node) +{ + CallLinkInfo::CallType callType; + bool isVarargs = false; + bool isForwardVarargs = false; + switch (node->op()) { + case Call: + callType = CallLinkInfo::Call; + break; + case Construct: + callType = CallLinkInfo::Construct; + break; + case CallVarargs: + callType = CallLinkInfo::CallVarargs; + isVarargs = true; + break; + case ConstructVarargs: + callType = CallLinkInfo::ConstructVarargs; + isVarargs = true; + break; + case CallForwardVarargs: + callType = CallLinkInfo::CallVarargs; + isForwardVarargs = true; + break; + case ConstructForwardVarargs: + callType = CallLinkInfo::ConstructVarargs; + isForwardVarargs = true; + break; + default: + DFG_CRASH(m_jit.graph(), node, "bad node type"); + break; + } + + Edge calleeEdge = m_jit.graph().child(node, 0); - m_jit.store32(MacroAssembler::TrustedImm32(numPassedArgs + dummyThisArgument), callFramePayloadSlot(JSStack::ArgumentCount)); - m_jit.store64(GPRInfo::callFrameRegister, callFrameSlot(JSStack::CallerFrame)); - m_jit.store64(calleeGPR, callFrameSlot(JSStack::Callee)); + // Gotta load the arguments somehow. Varargs is trickier. + if (isVarargs || isForwardVarargs) { + CallVarargsData* data = node->callVarargsData(); + + GPRReg resultGPR; + unsigned numUsedStackSlots = m_jit.graph().m_nextMachineLocal; + + if (isForwardVarargs) { + flushRegisters(); + use(node->child2()); + + GPRReg scratchGPR1; + GPRReg scratchGPR2; + GPRReg scratchGPR3; + + scratchGPR1 = JITCompiler::selectScratchGPR(); + scratchGPR2 = JITCompiler::selectScratchGPR(scratchGPR1); + scratchGPR3 = JITCompiler::selectScratchGPR(scratchGPR1, scratchGPR2); + + m_jit.move(TrustedImm32(numUsedStackSlots), scratchGPR2); + JITCompiler::JumpList slowCase; + emitSetupVarargsFrameFastCase(m_jit, scratchGPR2, scratchGPR1, scratchGPR2, scratchGPR3, node->child2()->origin.semantic.inlineCallFrame, data->firstVarArgOffset, slowCase); + JITCompiler::Jump done = m_jit.jump(); + slowCase.link(&m_jit); + callOperation(operationThrowStackOverflowForVarargs); + m_jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow); + done.link(&m_jit); + resultGPR = scratchGPR2; + } else { + GPRReg argumentsGPR; + GPRReg scratchGPR1; + GPRReg scratchGPR2; + GPRReg scratchGPR3; + + auto loadArgumentsGPR = [&] (GPRReg reservedGPR) { + if (reservedGPR != InvalidGPRReg) + lock(reservedGPR); + JSValueOperand arguments(this, node->child2()); + argumentsGPR = arguments.gpr(); + if (reservedGPR != InvalidGPRReg) + unlock(reservedGPR); + flushRegisters(); + + scratchGPR1 = JITCompiler::selectScratchGPR(argumentsGPR, reservedGPR); + scratchGPR2 = JITCompiler::selectScratchGPR(argumentsGPR, scratchGPR1, reservedGPR); + scratchGPR3 = JITCompiler::selectScratchGPR(argumentsGPR, scratchGPR1, scratchGPR2, reservedGPR); + }; + + loadArgumentsGPR(InvalidGPRReg); + + DFG_ASSERT(m_jit.graph(), node, isFlushed()); + + // Right now, arguments is in argumentsGPR and the register file is flushed. + callOperation(operationSizeFrameForVarargs, GPRInfo::returnValueGPR, argumentsGPR, numUsedStackSlots, data->firstVarArgOffset); + + // Now we have the argument count of the callee frame, but we've lost the arguments operand. + // Reconstruct the arguments operand while preserving the callee frame. + loadArgumentsGPR(GPRInfo::returnValueGPR); + m_jit.move(TrustedImm32(numUsedStackSlots), scratchGPR1); + emitSetVarargsFrame(m_jit, GPRInfo::returnValueGPR, false, scratchGPR1, scratchGPR1); + m_jit.addPtr(TrustedImm32(-(sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(void*)))), scratchGPR1, JITCompiler::stackPointerRegister); + + callOperation(operationSetupVarargsFrame, GPRInfo::returnValueGPR, scratchGPR1, argumentsGPR, data->firstVarArgOffset, GPRInfo::returnValueGPR); + resultGPR = GPRInfo::returnValueGPR; + } + + m_jit.addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), resultGPR, JITCompiler::stackPointerRegister); + + DFG_ASSERT(m_jit.graph(), node, isFlushed()); + + // We don't need the arguments array anymore. + if (isVarargs) + use(node->child2()); + + // Now set up the "this" argument. + JSValueOperand thisArgument(this, node->child3()); + GPRReg thisArgumentGPR = thisArgument.gpr(); + thisArgument.use(); + + m_jit.store64(thisArgumentGPR, JITCompiler::calleeArgumentSlot(0)); + } else { + // The call instruction's first child is the function; the subsequent children are the + // arguments. + int numPassedArgs = node->numChildren() - 1; + + m_jit.store32(MacroAssembler::TrustedImm32(numPassedArgs), JITCompiler::calleeFramePayloadSlot(JSStack::ArgumentCount)); - for (int i = 0; i < numPassedArgs; i++) { - Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i]; - JSValueOperand arg(this, argEdge); - GPRReg argGPR = arg.gpr(); - use(argEdge); + for (int i = 0; i < numPassedArgs; i++) { + Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i]; + JSValueOperand arg(this, argEdge); + GPRReg argGPR = arg.gpr(); + use(argEdge); - m_jit.store64(argGPR, argumentSlot(i + dummyThisArgument)); + m_jit.store64(argGPR, JITCompiler::calleeArgumentSlot(i)); + } } + JSValueOperand callee(this, calleeEdge); + GPRReg calleeGPR = callee.gpr(); + callee.use(); + m_jit.store64(calleeGPR, JITCompiler::calleeFrameSlot(JSStack::Callee)); + flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); JITCompiler::DataLabelPtr targetToCheck; - JITCompiler::JumpList slowPath; + JITCompiler::Jump slowPath; - CallBeginToken token; - m_jit.beginCall(node->codeOrigin, token); + m_jit.emitStoreCodeOrigin(node->origin.semantic); - m_jit.addPtr(TrustedImm32(m_jit.codeBlock()->m_numCalleeRegisters * sizeof(Register)), GPRInfo::callFrameRegister); + CallLinkInfo* callLinkInfo = m_jit.codeBlock()->addCallLinkInfo(); - slowPath.append(m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleeGPR, targetToCheck, MacroAssembler::TrustedImmPtr(0))); + slowPath = m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleeGPR, targetToCheck, MacroAssembler::TrustedImmPtr(0)); - m_jit.loadPtr(MacroAssembler::Address(calleeGPR, OBJECT_OFFSETOF(JSFunction, m_scope)), resultGPR); - m_jit.store64(resultGPR, MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast(sizeof(Register)) * JSStack::ScopeChain)); - - CodeOrigin codeOrigin = m_currentNode->codeOrigin; JITCompiler::Call fastCall = m_jit.nearCall(); - m_jit.notifyCall(fastCall, codeOrigin, token); - + JITCompiler::Jump done = m_jit.jump(); slowPath.link(&m_jit); - m_jit.move(calleeGPR, GPRInfo::nonArgGPR0); - m_jit.prepareForExceptionCheck(); + m_jit.move(calleeGPR, GPRInfo::regT0); // Callee needs to be in regT0 + m_jit.move(MacroAssembler::TrustedImmPtr(callLinkInfo), GPRInfo::regT2); // Link info needs to be in regT2 JITCompiler::Call slowCall = m_jit.nearCall(); - m_jit.notifyCall(slowCall, codeOrigin, token); done.link(&m_jit); @@ -808,189 +792,269 @@ void SpeculativeJIT::emitCall(Node* node) jsValueResult(resultGPR, m_currentNode, DataFormatJS, UseChildrenCalledExplicitly); - m_jit.addJSCall(fastCall, slowCall, targetToCheck, callType, calleeGPR, m_currentNode->codeOrigin); + callLinkInfo->setUpCall(callType, m_currentNode->origin.semantic, calleeGPR); + m_jit.addJSCall(fastCall, slowCall, targetToCheck, callLinkInfo); + + // If we were varargs, then after the calls are done, we need to reestablish our stack pointer. + if (isVarargs || isForwardVarargs) + m_jit.addPtr(TrustedImm32(m_jit.graph().stackPointerOffset() * sizeof(Register)), GPRInfo::callFrameRegister, JITCompiler::stackPointerRegister); } +// Clang should allow unreachable [[clang::fallthrough]] in template functions if any template expansion uses it +// http://llvm.org/bugs/show_bug.cgi?id=18619 +#if COMPILER(CLANG) && defined(__has_warning) +#pragma clang diagnostic push +#if __has_warning("-Wimplicit-fallthrough") +#pragma clang diagnostic ignored "-Wimplicit-fallthrough" +#endif +#endif template -GPRReg SpeculativeJIT::fillSpeculateIntInternal(Edge edge, DataFormat& returnFormat) +GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnFormat) { -#if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("SpecInt@%d ", edge->index()); -#endif AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32)); - value.filter(SpecInt32); + + m_interpreter.filter(value, SpecInt32); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + returnFormat = DataFormatInt32; + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; + GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if ((edge->hasConstant() && !isInt32Constant(edge.node())) || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - returnFormat = DataFormatInteger; - return allocate(); - } - GPRReg gpr = allocate(); if (edge->hasConstant()) { m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - ASSERT(isInt32Constant(edge.node())); - m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr); - info.fillInteger(*m_stream, gpr); - returnFormat = DataFormatInteger; + ASSERT(edge->isInt32Constant()); + m_jit.move(MacroAssembler::Imm32(edge->asInt32()), gpr); + info.fillInt32(*m_stream, gpr); + returnFormat = DataFormatInt32; return gpr; } DataFormat spillFormat = info.spillFormat(); - RELEASE_ASSERT((spillFormat & DataFormatJS) || spillFormat == DataFormatInteger); + DFG_ASSERT(m_jit.graph(), m_currentNode, (spillFormat & DataFormatJS) || spillFormat == DataFormatInt32); m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - if (spillFormat == DataFormatJSInteger || spillFormat == DataFormatInteger) { + if (spillFormat == DataFormatJSInt32 || spillFormat == DataFormatInt32) { // If we know this was spilled as an integer we can fill without checking. if (strict) { m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr); - info.fillInteger(*m_stream, gpr); - returnFormat = DataFormatInteger; + info.fillInt32(*m_stream, gpr); + returnFormat = DataFormatInt32; return gpr; } - if (spillFormat == DataFormatInteger) { + if (spillFormat == DataFormatInt32) { m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr); m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr); } else m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJSInteger); - returnFormat = DataFormatJSInteger; + info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); + returnFormat = DataFormatJSInt32; return gpr; } m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); // Fill as JSValue, and fall through. - info.fillJSValue(*m_stream, gpr, DataFormatJSInteger); + info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); m_gprs.unlock(gpr); + FALLTHROUGH; } case DataFormatJS: { + DFG_ASSERT(m_jit.graph(), m_currentNode, !(type & SpecInt52)); // Check the value is an integer. GPRReg gpr = info.gpr(); m_gprs.lock(gpr); if (type & ~SpecInt32) speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branch64(MacroAssembler::Below, gpr, GPRInfo::tagTypeNumberRegister)); - info.fillJSValue(*m_stream, gpr, DataFormatJSInteger); + info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); // If !strict we're done, return. if (!strict) { - returnFormat = DataFormatJSInteger; + returnFormat = DataFormatJSInt32; return gpr; } - // else fall through & handle as DataFormatJSInteger. + // else fall through & handle as DataFormatJSInt32. m_gprs.unlock(gpr); + FALLTHROUGH; } - case DataFormatJSInteger: { + case DataFormatJSInt32: { // In a strict fill we need to strip off the value tag. if (strict) { GPRReg gpr = info.gpr(); GPRReg result; // If the register has already been locked we need to take a copy. - // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInteger, not DataFormatJSInteger. + // If not, we'll zero extend in place, so mark on the info that this is now type DataFormatInt32, not DataFormatJSInt32. if (m_gprs.isLocked(gpr)) result = allocate(); else { m_gprs.lock(gpr); - info.fillInteger(*m_stream, gpr); + info.fillInt32(*m_stream, gpr); result = gpr; } m_jit.zeroExtend32ToPtr(gpr, result); - returnFormat = DataFormatInteger; + returnFormat = DataFormatInt32; return result; } GPRReg gpr = info.gpr(); m_gprs.lock(gpr); - returnFormat = DataFormatJSInteger; + returnFormat = DataFormatJSInt32; return gpr; } - case DataFormatInteger: { + case DataFormatInt32: { GPRReg gpr = info.gpr(); m_gprs.lock(gpr); - returnFormat = DataFormatInteger; + returnFormat = DataFormatInt32; return gpr; } - - case DataFormatDouble: - case DataFormatJSDouble: { - if (edge->hasConstant() && isInt32Constant(edge.node())) { - GPRReg gpr = allocate(); - ASSERT(isInt32Constant(edge.node())); - m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr); - returnFormat = DataFormatInteger; - return gpr; - } - } + + case DataFormatJSDouble: case DataFormatCell: case DataFormatBoolean: case DataFormatJSCell: - case DataFormatJSBoolean: { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - returnFormat = DataFormatInteger; - return allocate(); - } - + case DataFormatJSBoolean: + case DataFormatDouble: case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); + case DataFormatInt52: + case DataFormatStrictInt52: + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Corrupt data format"); return InvalidGPRReg; } } +#if COMPILER(CLANG) && defined(__has_warning) +#pragma clang diagnostic pop +#endif -GPRReg SpeculativeJIT::fillSpeculateInt(Edge edge, DataFormat& returnFormat) +GPRReg SpeculativeJIT::fillSpeculateInt32(Edge edge, DataFormat& returnFormat) { - return fillSpeculateIntInternal(edge, returnFormat); + return fillSpeculateInt32Internal(edge, returnFormat); } -GPRReg SpeculativeJIT::fillSpeculateIntStrict(Edge edge) +GPRReg SpeculativeJIT::fillSpeculateInt32Strict(Edge edge) { - DataFormat mustBeDataFormatInteger; - GPRReg result = fillSpeculateIntInternal(edge, mustBeDataFormatInteger); - RELEASE_ASSERT(mustBeDataFormatInteger == DataFormatInteger); + DataFormat mustBeDataFormatInt32; + GPRReg result = fillSpeculateInt32Internal(edge, mustBeDataFormatInt32); + DFG_ASSERT(m_jit.graph(), m_currentNode, mustBeDataFormatInt32 == DataFormatInt32); return result; } -FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge) +GPRReg SpeculativeJIT::fillSpeculateInt52(Edge edge, DataFormat desiredFormat) { -#if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("SpecDouble@%d ", edge->index()); -#endif + ASSERT(desiredFormat == DataFormatInt52 || desiredFormat == DataFormatStrictInt52); AbstractValue& value = m_state.forNode(edge); - SpeculatedType type = value.m_type; - ASSERT(edge.useKind() != KnownNumberUse || !(value.m_type & ~SpecNumber)); - value.filter(SpecNumber); + + m_interpreter.filter(value, SpecMachineInt); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + return allocate(); + } + + VirtualRegister virtualRegister = edge->virtualRegister(); + GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); + + switch (info.registerFormat()) { + case DataFormatNone: { + GPRReg gpr = allocate(); + + if (edge->hasConstant()) { + JSValue jsValue = edge->asJSValue(); + ASSERT(jsValue.isMachineInt()); + m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); + int64_t value = jsValue.asMachineInt(); + if (desiredFormat == DataFormatInt52) + value = value << JSValue::int52ShiftAmount; + m_jit.move(MacroAssembler::Imm64(value), gpr); + info.fillGPR(*m_stream, gpr, desiredFormat); + return gpr; + } + + DataFormat spillFormat = info.spillFormat(); + + DFG_ASSERT(m_jit.graph(), m_currentNode, spillFormat == DataFormatInt52 || spillFormat == DataFormatStrictInt52); + + m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); + + m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); + if (desiredFormat == DataFormatStrictInt52) { + if (spillFormat == DataFormatInt52) + m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); + info.fillStrictInt52(*m_stream, gpr); + return gpr; + } + if (spillFormat == DataFormatStrictInt52) + m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); + info.fillInt52(*m_stream, gpr); + return gpr; + } + + case DataFormatStrictInt52: { + GPRReg gpr = info.gpr(); + bool wasLocked = m_gprs.isLocked(gpr); + lock(gpr); + if (desiredFormat == DataFormatStrictInt52) + return gpr; + if (wasLocked) { + GPRReg result = allocate(); + m_jit.move(gpr, result); + unlock(gpr); + gpr = result; + } else + info.fillInt52(*m_stream, gpr); + m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); + return gpr; + } + + case DataFormatInt52: { + GPRReg gpr = info.gpr(); + bool wasLocked = m_gprs.isLocked(gpr); + lock(gpr); + if (desiredFormat == DataFormatInt52) + return gpr; + if (wasLocked) { + GPRReg result = allocate(); + m_jit.move(gpr, result); + unlock(gpr); + gpr = result; + } else + info.fillStrictInt52(*m_stream, gpr); + m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); + return gpr; + } + + default: + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); + return InvalidGPRReg; + } +} + +FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge) +{ + ASSERT(edge.useKind() == DoubleRepUse || edge.useKind() == DoubleRepRealUse || edge.useKind() == DoubleRepMachineIntUse); + ASSERT(edge->hasDoubleResult()); VirtualRegister virtualRegister = edge->virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; + GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); if (info.registerFormat() == DataFormatNone) { if (edge->hasConstant()) { GPRReg gpr = allocate(); - if (isInt32Constant(edge.node())) { - FPRReg fpr = fprAllocate(); - m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(static_cast(valueOfInt32Constant(edge.node())))), gpr); - m_jit.move64ToDouble(gpr, fpr); - unlock(gpr); - - m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); - info.fillDouble(*m_stream, fpr); - return fpr; - } - if (isNumberConstant(edge.node())) { + if (edge->isNumberConstant()) { FPRReg fpr = fprAllocate(); - m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(valueOfNumberConstant(edge.node()))), gpr); + m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(edge->asNumber())), gpr); m_jit.move64ToDouble(gpr, fpr); unlock(gpr); @@ -1003,160 +1067,59 @@ FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge) } DataFormat spillFormat = info.spillFormat(); - switch (spillFormat) { - case DataFormatDouble: { - FPRReg fpr = fprAllocate(); - m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr); - m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); - info.fillDouble(*m_stream, fpr); - return fpr; - } - - case DataFormatInteger: { - GPRReg gpr = allocate(); - - m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr); - info.fillInteger(*m_stream, gpr); - unlock(gpr); - break; - } - - default: - GPRReg gpr = allocate(); - - RELEASE_ASSERT(spillFormat & DataFormatJS); - m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - info.fillJSValue(*m_stream, gpr, spillFormat); - unlock(gpr); - break; + if (spillFormat != DataFormatDouble) { + DFG_CRASH( + m_jit.graph(), m_currentNode, toCString( + "Expected ", edge, " to have double format but instead it is spilled as ", + dataFormatToString(spillFormat)).data()); } + DFG_ASSERT(m_jit.graph(), m_currentNode, spillFormat == DataFormatDouble); + FPRReg fpr = fprAllocate(); + m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr); + m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); + info.fillDouble(*m_stream, fpr); + return fpr; } - switch (info.registerFormat()) { - case DataFormatNone: // Should have filled, above. - case DataFormatBoolean: // This type never occurs. - case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); - - case DataFormatCell: - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return fprAllocate(); - - case DataFormatJSCell: - case DataFormatJS: - case DataFormatJSBoolean: { - GPRReg jsValueGpr = info.gpr(); - m_gprs.lock(jsValueGpr); - FPRReg fpr = fprAllocate(); - GPRReg tempGpr = allocate(); - - JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, jsValueGpr, GPRInfo::tagTypeNumberRegister); - - if (type & ~SpecNumber) - speculationCheck(BadType, JSValueRegs(jsValueGpr), edge, m_jit.branchTest64(MacroAssembler::Zero, jsValueGpr, GPRInfo::tagTypeNumberRegister)); - - // First, if we get here we have a double encoded as a JSValue - m_jit.move(jsValueGpr, tempGpr); - unboxDouble(tempGpr, fpr); - JITCompiler::Jump hasUnboxedDouble = m_jit.jump(); - - // Finally, handle integers. - isInteger.link(&m_jit); - m_jit.convertInt32ToDouble(jsValueGpr, fpr); - hasUnboxedDouble.link(&m_jit); - - m_gprs.release(jsValueGpr); - m_gprs.unlock(jsValueGpr); - m_gprs.unlock(tempGpr); - m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); - info.fillDouble(*m_stream, fpr); - info.killSpilled(); - return fpr; - } - - case DataFormatJSInteger: - case DataFormatInteger: { - FPRReg fpr = fprAllocate(); - GPRReg gpr = info.gpr(); - m_gprs.lock(gpr); - m_jit.convertInt32ToDouble(gpr, fpr); - m_gprs.unlock(gpr); - return fpr; - } - - // Unbox the double - case DataFormatJSDouble: { - GPRReg gpr = info.gpr(); - FPRReg fpr = fprAllocate(); - if (m_gprs.isLocked(gpr)) { - // Make sure we don't trample gpr if it is in use. - GPRReg temp = allocate(); - m_jit.move(gpr, temp); - unboxDouble(temp, fpr); - unlock(temp); - } else - unboxDouble(gpr, fpr); - - m_gprs.release(gpr); - m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); - - info.fillDouble(*m_stream, fpr); - return fpr; - } - - case DataFormatDouble: { - FPRReg fpr = info.fpr(); - m_fprs.lock(fpr); - return fpr; - } - - default: - RELEASE_ASSERT_NOT_REACHED(); - return InvalidFPRReg; - } + DFG_ASSERT(m_jit.graph(), m_currentNode, info.registerFormat() == DataFormatDouble); + FPRReg fpr = info.fpr(); + m_fprs.lock(fpr); + return fpr; } GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) { -#if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("SpecCell@%d ", edge->index()); -#endif AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; ASSERT((edge.useKind() != KnownCellUse && edge.useKind() != KnownStringUse) || !(value.m_type & ~SpecCell)); - value.filter(SpecCell); + + m_interpreter.filter(value, SpecCell); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; + GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if (info.spillFormat() == DataFormatInteger || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - GPRReg gpr = allocate(); if (edge->hasConstant()) { - JSValue jsValue = valueOfJSConstant(edge.node()); - if (jsValue.isCell()) { - m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJSCell); - return gpr; - } - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + JSValue jsValue = edge->asJSValue(); + m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); + info.fillJSValue(*m_stream, gpr, DataFormatJSCell); return gpr; } - RELEASE_ASSERT(info.spillFormat() & DataFormatJS); + m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); info.fillJSValue(*m_stream, gpr, DataFormatJS); if (type & ~SpecCell) - speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagMaskRegister)); + speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchIfNotCell(JSValueRegs(gpr))); info.fillJSValue(*m_stream, gpr, DataFormatJSCell); return gpr; } @@ -1165,6 +1128,11 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) case DataFormatJSCell: { GPRReg gpr = info.gpr(); m_gprs.lock(gpr); + if (!ASSERT_DISABLED) { + MacroAssembler::Jump checkCell = m_jit.branchIfCell(JSValueRegs(gpr)); + m_jit.abortWithReason(DFGIsNotCell); + checkCell.link(&m_jit); + } return gpr; } @@ -1172,62 +1140,54 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) GPRReg gpr = info.gpr(); m_gprs.lock(gpr); if (type & ~SpecCell) - speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagMaskRegister)); + speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchIfNotCell(JSValueRegs(gpr))); info.fillJSValue(*m_stream, gpr, DataFormatJSCell); return gpr; } - case DataFormatJSInteger: - case DataFormatInteger: + case DataFormatJSInt32: + case DataFormatInt32: case DataFormatJSDouble: - case DataFormatDouble: case DataFormatJSBoolean: - case DataFormatBoolean: { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - + case DataFormatBoolean: + case DataFormatDouble: case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); + case DataFormatInt52: + case DataFormatStrictInt52: + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Corrupt data format"); return InvalidGPRReg; } } GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge) { -#if DFG_ENABLE(DEBUG_VERBOSE) - dataLogF("SpecBool@%d ", edge->index()); -#endif AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; - value.filter(SpecBoolean); + + m_interpreter.filter(value, SpecBoolean); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); - GenerationInfo& info = m_generationInfo[virtualRegister]; + GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if (info.spillFormat() == DataFormatInteger || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - GPRReg gpr = allocate(); if (edge->hasConstant()) { - JSValue jsValue = valueOfJSConstant(edge.node()); - if (jsValue.isBoolean()) { - m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean); - return gpr; - } - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + JSValue jsValue = edge->asJSValue(); + m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); + info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean); return gpr; } - RELEASE_ASSERT(info.spillFormat() & DataFormatJS); + DFG_ASSERT(m_jit.graph(), m_currentNode, info.spillFormat() & DataFormatJS); m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); @@ -1260,93 +1220,70 @@ GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge) return gpr; } - case DataFormatJSInteger: - case DataFormatInteger: + case DataFormatJSInt32: + case DataFormatInt32: case DataFormatJSDouble: - case DataFormatDouble: case DataFormatJSCell: - case DataFormatCell: { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - + case DataFormatCell: + case DataFormatDouble: case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); + case DataFormatInt52: + case DataFormatStrictInt52: + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Corrupt data format"); return InvalidGPRReg; } } -JITCompiler::Jump SpeculativeJIT::convertToDouble(GPRReg value, FPRReg result, GPRReg tmp) +void SpeculativeJIT::compileBaseValueStoreBarrier(Edge& baseEdge, Edge& valueEdge) { - JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, value, GPRInfo::tagTypeNumberRegister); - - JITCompiler::Jump notNumber = m_jit.branchTest64(MacroAssembler::Zero, value, GPRInfo::tagTypeNumberRegister); - - m_jit.move(value, tmp); - unboxDouble(tmp, result); - - JITCompiler::Jump done = m_jit.jump(); - - isInteger.link(&m_jit); - - m_jit.convertInt32ToDouble(value, result); - - done.link(&m_jit); +#if ENABLE(GGC) + ASSERT(!isKnownNotCell(valueEdge.node())); + + SpeculateCellOperand base(this, baseEdge); + JSValueOperand value(this, valueEdge); + GPRTemporary scratch1(this); + GPRTemporary scratch2(this); - return notNumber; + writeBarrier(base.gpr(), value.gpr(), valueEdge, scratch1.gpr(), scratch2.gpr()); +#else + UNUSED_PARAM(baseEdge); + UNUSED_PARAM(valueEdge); +#endif } void SpeculativeJIT::compileObjectEquality(Node* node) { SpeculateCellOperand op1(this, node->child1()); SpeculateCellOperand op2(this, node->child2()); - GPRTemporary result(this, op1); + GPRTemporary result(this, Reuse, op1); GPRReg op1GPR = op1.gpr(); GPRReg op2GPR = op2.gpr(); GPRReg resultGPR = result.gpr(); - if (m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) { - m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); + if (masqueradesAsUndefinedWatchpointIsStillValid()) { DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchIfNotObject(op1GPR)); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchIfNotObject(op2GPR)); } else { - GPRTemporary structure(this); - GPRReg structureGPR = structure.gpr(); - - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchIfNotObject(op1GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); - m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchIfNotObject(op2GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } @@ -1360,6 +1297,47 @@ void SpeculativeJIT::compileObjectEquality(Node* node) jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean); } +void SpeculativeJIT::compileObjectStrictEquality(Edge objectChild, Edge otherChild) +{ + SpeculateCellOperand op1(this, objectChild); + JSValueOperand op2(this, otherChild); + GPRTemporary result(this); + + GPRReg op1GPR = op1.gpr(); + GPRReg op2GPR = op2.gpr(); + GPRReg resultGPR = result.gpr(); + + DFG_TYPE_CHECK(JSValueSource::unboxedCell(op1GPR), objectChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); + + // At this point we know that we can perform a straight-forward equality comparison on pointer + // values because we are doing strict equality. + m_jit.compare64(MacroAssembler::Equal, op1GPR, op2GPR, resultGPR); + m_jit.or32(TrustedImm32(ValueFalse), resultGPR); + jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean); +} + +void SpeculativeJIT::compilePeepHoleObjectStrictEquality(Edge objectChild, Edge otherChild, Node* branchNode) +{ + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; + + SpeculateCellOperand op1(this, objectChild); + JSValueOperand op2(this, otherChild); + + GPRReg op1GPR = op1.gpr(); + GPRReg op2GPR = op2.gpr(); + + DFG_TYPE_CHECK(JSValueSource::unboxedCell(op1GPR), objectChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); + + if (taken == nextBlock()) { + branchPtr(MacroAssembler::NotEqual, op1GPR, op2GPR, notTaken); + jump(taken); + } else { + branchPtr(MacroAssembler::Equal, op1GPR, op2GPR, taken); + jump(notTaken); + } +} + void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild) { SpeculateCellOperand op1(this, leftChild); @@ -1369,64 +1347,38 @@ void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge r GPRReg op1GPR = op1.gpr(); GPRReg op2GPR = op2.gpr(); GPRReg resultGPR = result.gpr(); - GPRTemporary structure; - GPRReg structureGPR = InvalidGPRReg; - bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid(); - - if (!masqueradesAsUndefinedWatchpointValid) { - // The masquerades as undefined case will use the structure register, so allocate it here. - // Do this at the top of the function to avoid branching around a register allocation. - GPRTemporary realStructure(this); - structure.adopt(realStructure); - structureGPR = structure.gpr(); - } + bool masqueradesAsUndefinedWatchpointValid = + masqueradesAsUndefinedWatchpointIsStillValid(); if (masqueradesAsUndefinedWatchpointValid) { - m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } // It seems that most of the time when programs do a == b where b may be either null/undefined // or an object, b is usually an object. Balance the branches to make that case fast. - MacroAssembler::Jump rightNotCell = - m_jit.branchTest64(MacroAssembler::NonZero, op2GPR, GPRInfo::tagMaskRegister); + MacroAssembler::Jump rightNotCell = m_jit.branchIfNotCell(JSValueRegs(op2GPR)); // We know that within this branch, rightChild must be a cell. - if (masqueradesAsUndefinedWatchpointValid) { - m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); + if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2GPR)); speculationCheck(BadType, JSValueRegs(op2GPR), rightChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } @@ -1463,8 +1415,8 @@ void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge r void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild, Node* branchNode) { - BlockIndex taken = branchNode->takenBlockIndex(); - BlockIndex notTaken = branchNode->notTakenBlockIndex(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; SpeculateCellOperand op1(this, leftChild); JSValueOperand op2(this, rightChild, ManualOperandSpeculation); @@ -1473,64 +1425,38 @@ void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild GPRReg op1GPR = op1.gpr(); GPRReg op2GPR = op2.gpr(); GPRReg resultGPR = result.gpr(); - GPRTemporary structure; - GPRReg structureGPR = InvalidGPRReg; - bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid(); - - if (!masqueradesAsUndefinedWatchpointValid) { - // The masquerades as undefined case will use the structure register, so allocate it here. - // Do this at the top of the function to avoid branching around a register allocation. - GPRTemporary realStructure(this); - structure.adopt(realStructure); - structureGPR = structure.gpr(); - } + bool masqueradesAsUndefinedWatchpointValid = + masqueradesAsUndefinedWatchpointIsStillValid(); if (masqueradesAsUndefinedWatchpointValid) { - m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } // It seems that most of the time when programs do a == b where b may be either null/undefined // or an object, b is usually an object. Balance the branches to make that case fast. - MacroAssembler::Jump rightNotCell = - m_jit.branchTest64(MacroAssembler::NonZero, op2GPR, GPRInfo::tagMaskRegister); + MacroAssembler::Jump rightNotCell = m_jit.branchIfNotCell(JSValueRegs(op2GPR)); // We know that within this branch, rightChild must be a cell. if (masqueradesAsUndefinedWatchpointValid) { - m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); DFG_TYPE_CHECK( - JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2GPR)); speculationCheck(BadType, JSValueRegs(op2GPR), rightChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } @@ -1559,11 +1485,11 @@ void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild jump(notTaken); } -void SpeculativeJIT::compileIntegerCompare(Node* node, MacroAssembler::RelationalCondition condition) +void SpeculativeJIT::compileInt32Compare(Node* node, MacroAssembler::RelationalCondition condition) { - SpeculateIntegerOperand op1(this, node->child1()); - SpeculateIntegerOperand op2(this, node->child2()); - GPRTemporary result(this, op1, op2); + SpeculateInt32Operand op1(this, node->child1()); + SpeculateInt32Operand op2(this, node->child2()); + GPRTemporary result(this, Reuse, op1, op2); m_jit.compare32(condition, op1.gpr(), op2.gpr(), result.gpr()); @@ -1572,6 +1498,40 @@ void SpeculativeJIT::compileIntegerCompare(Node* node, MacroAssembler::Relationa jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean); } +void SpeculativeJIT::compileInt52Compare(Node* node, MacroAssembler::RelationalCondition condition) +{ + SpeculateWhicheverInt52Operand op1(this, node->child1()); + SpeculateWhicheverInt52Operand op2(this, node->child2(), op1); + GPRTemporary result(this, Reuse, op1, op2); + + m_jit.compare64(condition, op1.gpr(), op2.gpr(), result.gpr()); + + // If we add a DataFormatBool, we should use it here. + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean); +} + +void SpeculativeJIT::compilePeepHoleInt52Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition) +{ + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; + + // The branch instruction will branch to the taken block. + // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through. + if (taken == nextBlock()) { + condition = JITCompiler::invert(condition); + BasicBlock* tmp = taken; + taken = notTaken; + notTaken = tmp; + } + + SpeculateWhicheverInt52Operand op1(this, node->child1()); + SpeculateWhicheverInt52Operand op2(this, node->child2(), op1); + + branch64(condition, op1.gpr(), op2.gpr(), taken); + jump(notTaken); +} + void SpeculativeJIT::compileDoubleCompare(Node* node, MacroAssembler::DoubleCondition condition) { SpeculateDoubleOperand op1(this, node->child1()); @@ -1586,25 +1546,6 @@ void SpeculativeJIT::compileDoubleCompare(Node* node, MacroAssembler::DoubleCond jsValueResult(result.gpr(), node, DataFormatJSBoolean); } -void SpeculativeJIT::compileValueAdd(Node* node) -{ - JSValueOperand op1(this, node->child1()); - JSValueOperand op2(this, node->child2()); - - GPRReg op1GPR = op1.gpr(); - GPRReg op2GPR = op2.gpr(); - - flushRegisters(); - - GPRResult result(this); - if (isKnownNotNumber(node->child1().node()) || isKnownNotNumber(node->child2().node())) - callOperation(operationValueAddNotNumber, result.gpr(), op1GPR, op2GPR); - else - callOperation(operationValueAdd, result.gpr(), op1GPR, op2GPR); - - jsValueResult(result.gpr(), node); -} - void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse) { JSValueOperand value(this, nodeUse, ManualOperandSpeculation); @@ -1613,45 +1554,43 @@ void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse) GPRReg resultGPR = result.gpr(); GPRTemporary structure; GPRReg structureGPR = InvalidGPRReg; + GPRTemporary scratch; + GPRReg scratchGPR = InvalidGPRReg; - bool masqueradesAsUndefinedWatchpointValid = m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid(); + bool masqueradesAsUndefinedWatchpointValid = + masqueradesAsUndefinedWatchpointIsStillValid(); if (!masqueradesAsUndefinedWatchpointValid) { // The masquerades as undefined case will use the structure register, so allocate it here. // Do this at the top of the function to avoid branching around a register allocation. GPRTemporary realStructure(this); + GPRTemporary realScratch(this); structure.adopt(realStructure); + scratch.adopt(realScratch); structureGPR = structure.gpr(); + scratchGPR = scratch.gpr(); } - MacroAssembler::Jump notCell = m_jit.branchTest64(MacroAssembler::NonZero, valueGPR, GPRInfo::tagMaskRegister); + MacroAssembler::Jump notCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); if (masqueradesAsUndefinedWatchpointValid) { - m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); DFG_TYPE_CHECK( - JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(valueGPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(valueGPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(valueGPR, JSCell::structureOffset()), structureGPR); - DFG_TYPE_CHECK( - JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(valueGPR)); MacroAssembler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8( MacroAssembler::Zero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(valueGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined)); + m_jit.emitLoadStructure(valueGPR, structureGPR, scratchGPR); speculationCheck(BadType, JSValueRegs(valueGPR), nodeUse, m_jit.branchPtr( MacroAssembler::Equal, MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()), - MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)))); + MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)))); isNotMasqueradesAsUndefined.link(&m_jit); } @@ -1685,15 +1624,15 @@ void SpeculativeJIT::compileLogicalNot(Node* node) } case Int32Use: { - SpeculateIntegerOperand value(this, node->child1()); - GPRTemporary result(this, value); + SpeculateInt32Operand value(this, node->child1()); + GPRTemporary result(this, Reuse, value); m_jit.compare32(MacroAssembler::Equal, value.gpr(), MacroAssembler::TrustedImm32(0), result.gpr()); m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); jsValueResult(result.gpr(), node, DataFormatJSBoolean); return; } - case NumberUse: { + case DoubleRepUse: { SpeculateDoubleOperand value(this, node->child1()); FPRTemporary scratch(this); GPRTemporary result(this); @@ -1708,7 +1647,7 @@ void SpeculativeJIT::compileLogicalNot(Node* node) case BooleanUse: { if (!needsTypeCheck(node->child1(), SpecBoolean)) { SpeculateBooleanOperand value(this, node->child1()); - GPRTemporary result(this, value); + GPRTemporary result(this, Reuse, value); m_jit.move(value.gpr(), result.gpr()); m_jit.xor64(TrustedImm32(true), result.gpr()); @@ -1746,51 +1685,55 @@ void SpeculativeJIT::compileLogicalNot(Node* node) JITCompiler::Jump slowCase = m_jit.branchTest64(JITCompiler::NonZero, resultGPR, TrustedImm32(static_cast(~1))); addSlowPathGenerator( - slowPathCall(slowCase, this, dfgConvertJSValueToBoolean, resultGPR, arg1GPR)); + slowPathCall(slowCase, this, operationConvertJSValueToBoolean, resultGPR, arg1GPR)); m_jit.xor64(TrustedImm32(static_cast(ValueTrue)), resultGPR); jsValueResult(resultGPR, node, DataFormatJSBoolean, UseChildrenCalledExplicitly); return; } - + case StringUse: + return compileStringZeroLength(node); + default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } } -void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BlockIndex taken, BlockIndex notTaken) +void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BasicBlock* taken, BasicBlock* notTaken) { JSValueOperand value(this, nodeUse, ManualOperandSpeculation); GPRTemporary scratch(this); + GPRTemporary structure; GPRReg valueGPR = value.gpr(); GPRReg scratchGPR = scratch.gpr(); - - MacroAssembler::Jump notCell = m_jit.branchTest64(MacroAssembler::NonZero, valueGPR, GPRInfo::tagMaskRegister); - if (m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) { - m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); + GPRReg structureGPR = InvalidGPRReg; + + if (!masqueradesAsUndefinedWatchpointIsStillValid()) { + GPRTemporary realStructure(this); + structure.adopt(realStructure); + structureGPR = structure.gpr(); + } + MacroAssembler::Jump notCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); + if (masqueradesAsUndefinedWatchpointIsStillValid()) { DFG_TYPE_CHECK( - JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(valueGPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(valueGPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(valueGPR, JSCell::structureOffset()), scratchGPR); - DFG_TYPE_CHECK( - JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - scratchGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(valueGPR)); - JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::Zero, MacroAssembler::Address(scratchGPR, Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)); + JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::Zero, + MacroAssembler::Address(valueGPR, JSCell::typeInfoFlagsOffset()), + TrustedImm32(MasqueradesAsUndefined)); + m_jit.emitLoadStructure(valueGPR, structureGPR, scratchGPR); speculationCheck(BadType, JSValueRegs(valueGPR), nodeUse, m_jit.branchPtr( MacroAssembler::Equal, - MacroAssembler::Address(scratchGPR, Structure::globalObjectOffset()), - MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)))); + MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()), + MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)))); isNotMasqueradesAsUndefined.link(&m_jit); } @@ -1812,8 +1755,8 @@ void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BlockIndex taken, Blo void SpeculativeJIT::emitBranch(Node* node) { - BlockIndex taken = node->takenBlockIndex(); - BlockIndex notTaken = node->notTakenBlockIndex(); + BasicBlock* taken = node->branchData()->taken.block; + BasicBlock* notTaken = node->branchData()->notTaken.block; switch (node->child1().useKind()) { case ObjectOrOtherUse: { @@ -1822,18 +1765,18 @@ void SpeculativeJIT::emitBranch(Node* node) } case Int32Use: - case NumberUse: { + case DoubleRepUse: { if (node->child1().useKind() == Int32Use) { bool invert = false; if (taken == nextBlock()) { invert = true; - BlockIndex tmp = taken; + BasicBlock* tmp = taken; taken = notTaken; notTaken = tmp; } - SpeculateIntegerOperand value(this, node->child1()); + SpeculateInt32Operand value(this, node->child1()); branchTest32(invert ? MacroAssembler::Zero : MacroAssembler::NonZero, value.gpr(), taken); } else { SpeculateDoubleOperand value(this, node->child1()); @@ -1847,6 +1790,11 @@ void SpeculativeJIT::emitBranch(Node* node) return; } + case StringUse: { + emitStringBranch(node->child1(), taken, notTaken); + return; + } + case UntypedUse: case BooleanUse: { JSValueOperand value(this, node->child1(), ManualOperandSpeculation); @@ -1858,7 +1806,7 @@ void SpeculativeJIT::emitBranch(Node* node) if (taken == nextBlock()) { condition = MacroAssembler::Zero; - BlockIndex tmp = taken; + BasicBlock* tmp = taken; taken = notTaken; notTaken = tmp; } @@ -1889,7 +1837,7 @@ void SpeculativeJIT::emitBranch(Node* node) value.use(); silentSpillAllRegisters(resultGPR); - callOperation(dfgConvertJSValueToBoolean, resultGPR, valueGPR); + callOperation(operationConvertJSValueToBoolean, resultGPR, valueGPR); silentFillAllRegisters(resultGPR); branchTest32(MacroAssembler::NonZero, resultGPR, taken); @@ -1901,7 +1849,7 @@ void SpeculativeJIT::emitBranch(Node* node) } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad use kind"); } } @@ -1915,166 +1863,187 @@ void SpeculativeJIT::compile(Node* node) switch (op) { case JSConstant: + case DoubleConstant: + case Int52Constant: + case PhantomDirectArguments: + case PhantomClonedArguments: initConstantInfo(node); break; - case PhantomArguments: - initConstantInfo(node); - break; - - case WeakJSConstant: - m_jit.addWeakReference(node->weakConstant()); - initConstantInfo(node); - break; - case Identity: { - // CSE should always eliminate this. - RELEASE_ASSERT_NOT_REACHED(); + speculate(node, node->child1()); + switch (node->child1().useKind()) { + case DoubleRepUse: + case DoubleRepRealUse: + case DoubleRepMachineIntUse: { + SpeculateDoubleOperand op(this, node->child1()); + FPRTemporary scratch(this, op); + m_jit.moveDouble(op.fpr(), scratch.fpr()); + doubleResult(scratch.fpr(), node); + break; + } + case Int52RepUse: { + SpeculateInt52Operand op(this, node->child1()); + GPRTemporary result(this, Reuse, op); + m_jit.move(op.gpr(), result.gpr()); + int52Result(result.gpr(), node); + break; + } + default: { + JSValueOperand op(this, node->child1()); + GPRTemporary result(this, Reuse, op); + m_jit.move(op.gpr(), result.gpr()); + jsValueResult(result.gpr(), node); + break; + } + } // switch break; } case GetLocal: { - SpeculatedType prediction = node->variableAccessData()->prediction(); AbstractValue& value = m_state.variables().operand(node->local()); - // If we have no prediction for this local, then don't attempt to compile. - if (prediction == SpecNone) { - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); - break; - } - // If the CFA is tracking this variable and it found that the variable // cannot have been assigned, then don't attempt to proceed. if (value.isClear()) { - // FIXME: We should trap instead. - // https://bugs.webkit.org/show_bug.cgi?id=110383 - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); + m_compileOkay = false; break; } - if (node->variableAccessData()->shouldUseDoubleFormat()) { + switch (node->variableAccessData()->flushFormat()) { + case FlushedDouble: { FPRTemporary result(this); - m_jit.loadDouble(JITCompiler::addressFor(node->local()), result.fpr()); + m_jit.loadDouble(JITCompiler::addressFor(node->machineLocal()), result.fpr()); VirtualRegister virtualRegister = node->virtualRegister(); m_fprs.retain(result.fpr(), virtualRegister, SpillOrderDouble); - m_generationInfo[virtualRegister].initDouble(node, node->refCount(), result.fpr()); + generationInfoFromVirtualRegister(virtualRegister).initDouble(node, node->refCount(), result.fpr()); break; } - if (isInt32Speculation(value.m_type)) { + case FlushedInt32: { GPRTemporary result(this); - m_jit.load32(JITCompiler::payloadFor(node->local()), result.gpr()); + m_jit.load32(JITCompiler::payloadFor(node->machineLocal()), result.gpr()); - // Like integerResult, but don't useChildren - our children are phi nodes, + // Like int32Result, but don't useChildren - our children are phi nodes, // and don't represent values within this dataflow with virtual registers. VirtualRegister virtualRegister = node->virtualRegister(); m_gprs.retain(result.gpr(), virtualRegister, SpillOrderInteger); - m_generationInfo[virtualRegister].initInteger(node, node->refCount(), result.gpr()); + generationInfoFromVirtualRegister(virtualRegister).initInt32(node, node->refCount(), result.gpr()); + break; + } + + case FlushedInt52: { + GPRTemporary result(this); + m_jit.load64(JITCompiler::addressFor(node->machineLocal()), result.gpr()); + + VirtualRegister virtualRegister = node->virtualRegister(); + m_gprs.retain(result.gpr(), virtualRegister, SpillOrderJS); + generationInfoFromVirtualRegister(virtualRegister).initInt52(node, node->refCount(), result.gpr()); + break; + } + + default: + GPRTemporary result(this); + m_jit.load64(JITCompiler::addressFor(node->machineLocal()), result.gpr()); + + // Like jsValueResult, but don't useChildren - our children are phi nodes, + // and don't represent values within this dataflow with virtual registers. + VirtualRegister virtualRegister = node->virtualRegister(); + m_gprs.retain(result.gpr(), virtualRegister, SpillOrderJS); + + DataFormat format; + if (isCellSpeculation(value.m_type)) + format = DataFormatJSCell; + else if (isBooleanSpeculation(value.m_type)) + format = DataFormatJSBoolean; + else + format = DataFormatJS; + + generationInfoFromVirtualRegister(virtualRegister).initJSValue(node, node->refCount(), result.gpr(), format); break; } - - GPRTemporary result(this); - m_jit.load64(JITCompiler::addressFor(node->local()), result.gpr()); - - // Like jsValueResult, but don't useChildren - our children are phi nodes, - // and don't represent values within this dataflow with virtual registers. - VirtualRegister virtualRegister = node->virtualRegister(); - m_gprs.retain(result.gpr(), virtualRegister, SpillOrderJS); - - DataFormat format; - if (isCellSpeculation(value.m_type)) - format = DataFormatJSCell; - else if (isBooleanSpeculation(value.m_type)) - format = DataFormatJSBoolean; - else - format = DataFormatJS; - - m_generationInfo[virtualRegister].initJSValue(node, node->refCount(), result.gpr(), format); break; } case GetLocalUnlinked: { GPRTemporary result(this); - m_jit.load64(JITCompiler::addressFor(node->unlinkedLocal()), result.gpr()); + m_jit.load64(JITCompiler::addressFor(node->unlinkedMachineLocal()), result.gpr()); jsValueResult(result.gpr(), node); break; } - case MovHintAndCheck: { - compileMovHintAndCheck(node); + case MovHint: { + compileMovHint(m_currentNode); + noResult(node); break; } - case InlineStart: { - compileInlineStart(node); - break; - } - - case MovHint: case ZombieHint: { - RELEASE_ASSERT_NOT_REACHED(); + recordSetLocal(m_currentNode->unlinkedLocal(), VirtualRegister(), DataFormatDead); + noResult(node); break; } - - case SetLocal: { - // SetLocal doubles as a hint as to where a node will be stored and - // as a speculation point. So before we speculate make sure that we - // know where the child of this node needs to go in the virtual - // stack. - compileMovHint(node); - - if (node->variableAccessData()->shouldUnboxIfPossible()) { - if (node->variableAccessData()->shouldUseDoubleFormat()) { - SpeculateDoubleOperand value(this, node->child1()); - m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->local())); - noResult(node); - // Indicate that it's no longer necessary to retrieve the value of - // this bytecode variable from registers or other locations in the stack, - // but that it is stored as a double. - recordSetLocal(node->local(), ValueSource(DoubleInJSStack)); - break; - } - SpeculatedType predictedType = node->variableAccessData()->argumentAwarePrediction(); - if (isInt32Speculation(predictedType)) { - SpeculateIntegerOperand value(this, node->child1()); - m_jit.store32(value.gpr(), JITCompiler::payloadFor(node->local())); - noResult(node); - recordSetLocal(node->local(), ValueSource(Int32InJSStack)); - break; - } - if (isCellSpeculation(predictedType)) { - SpeculateCellOperand cell(this, node->child1()); - GPRReg cellGPR = cell.gpr(); - m_jit.store64(cellGPR, JITCompiler::addressFor(node->local())); - noResult(node); - recordSetLocal(node->local(), ValueSource(CellInJSStack)); - break; - } - if (isBooleanSpeculation(predictedType)) { - SpeculateBooleanOperand boolean(this, node->child1()); - m_jit.store64(boolean.gpr(), JITCompiler::addressFor(node->local())); - noResult(node); - recordSetLocal(node->local(), ValueSource(BooleanInJSStack)); - break; - } + case SetLocal: { + switch (node->variableAccessData()->flushFormat()) { + case FlushedDouble: { + SpeculateDoubleOperand value(this, node->child1()); + m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->machineLocal())); + noResult(node); + // Indicate that it's no longer necessary to retrieve the value of + // this bytecode variable from registers or other locations in the stack, + // but that it is stored as a double. + recordSetLocal(DataFormatDouble); + break; + } + + case FlushedInt32: { + SpeculateInt32Operand value(this, node->child1()); + m_jit.store32(value.gpr(), JITCompiler::payloadFor(node->machineLocal())); + noResult(node); + recordSetLocal(DataFormatInt32); + break; + } + + case FlushedInt52: { + SpeculateInt52Operand value(this, node->child1()); + m_jit.store64(value.gpr(), JITCompiler::addressFor(node->machineLocal())); + noResult(node); + recordSetLocal(DataFormatInt52); + break; + } + + case FlushedCell: { + SpeculateCellOperand cell(this, node->child1()); + GPRReg cellGPR = cell.gpr(); + m_jit.store64(cellGPR, JITCompiler::addressFor(node->machineLocal())); + noResult(node); + recordSetLocal(DataFormatCell); + break; + } + + case FlushedBoolean: { + SpeculateBooleanOperand boolean(this, node->child1()); + m_jit.store64(boolean.gpr(), JITCompiler::addressFor(node->machineLocal())); + noResult(node); + recordSetLocal(DataFormatBoolean); + break; + } + + case FlushedJSValue: { + JSValueOperand value(this, node->child1()); + m_jit.store64(value.gpr(), JITCompiler::addressFor(node->machineLocal())); + noResult(node); + recordSetLocal(dataFormatFor(node->variableAccessData()->flushFormat())); + break; + } + + default: + DFG_CRASH(m_jit.graph(), node, "Bad flush format"); + break; } - - JSValueOperand value(this, node->child1()); - m_jit.store64(value.gpr(), JITCompiler::addressFor(node->local())); - noResult(node); - - recordSetLocal(node->local(), ValueSource(ValueInJSStack)); - - // If we're storing an arguments object that has been optimized away, - // our variable event stream for OSR exit now reflects the optimized - // value (JSValue()). On the slow path, we want an arguments object - // instead. We add an additional move hint to show OSR exit that it - // needs to reconstruct the arguments object. - if (node->child1()->op() == PhantomArguments) - compileMovHint(node); break; } @@ -2084,59 +2053,60 @@ void SpeculativeJIT::compile(Node* node) // But it may be profitable to use this as a hook to run speculation checks // on arguments, thereby allowing us to trivially eliminate such checks if // the argument is not used. + recordSetLocal(dataFormatFor(node->variableAccessData()->flushFormat())); break; case BitAnd: case BitOr: case BitXor: - if (isInt32Constant(node->child1().node())) { - SpeculateIntegerOperand op2(this, node->child2()); - GPRTemporary result(this, op2); + if (node->child1()->isInt32Constant()) { + SpeculateInt32Operand op2(this, node->child2()); + GPRTemporary result(this, Reuse, op2); - bitOp(op, valueOfInt32Constant(node->child1().node()), op2.gpr(), result.gpr()); + bitOp(op, node->child1()->asInt32(), op2.gpr(), result.gpr()); - integerResult(result.gpr(), node); - } else if (isInt32Constant(node->child2().node())) { - SpeculateIntegerOperand op1(this, node->child1()); - GPRTemporary result(this, op1); + int32Result(result.gpr(), node); + } else if (node->child2()->isInt32Constant()) { + SpeculateInt32Operand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); - bitOp(op, valueOfInt32Constant(node->child2().node()), op1.gpr(), result.gpr()); + bitOp(op, node->child2()->asInt32(), op1.gpr(), result.gpr()); - integerResult(result.gpr(), node); + int32Result(result.gpr(), node); } else { - SpeculateIntegerOperand op1(this, node->child1()); - SpeculateIntegerOperand op2(this, node->child2()); - GPRTemporary result(this, op1, op2); + SpeculateInt32Operand op1(this, node->child1()); + SpeculateInt32Operand op2(this, node->child2()); + GPRTemporary result(this, Reuse, op1, op2); GPRReg reg1 = op1.gpr(); GPRReg reg2 = op2.gpr(); bitOp(op, reg1, reg2, result.gpr()); - integerResult(result.gpr(), node); + int32Result(result.gpr(), node); } break; case BitRShift: case BitLShift: case BitURShift: - if (isInt32Constant(node->child2().node())) { - SpeculateIntegerOperand op1(this, node->child1()); - GPRTemporary result(this, op1); + if (node->child2()->isInt32Constant()) { + SpeculateInt32Operand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); - shiftOp(op, op1.gpr(), valueOfInt32Constant(node->child2().node()) & 0x1f, result.gpr()); + shiftOp(op, op1.gpr(), node->child2()->asInt32() & 0x1f, result.gpr()); - integerResult(result.gpr(), node); + int32Result(result.gpr(), node); } else { // Do not allow shift amount to be used as the result, MacroAssembler does not permit this. - SpeculateIntegerOperand op1(this, node->child1()); - SpeculateIntegerOperand op2(this, node->child2()); - GPRTemporary result(this, op1); + SpeculateInt32Operand op1(this, node->child1()); + SpeculateInt32Operand op2(this, node->child2()); + GPRTemporary result(this, Reuse, op1); GPRReg reg1 = op1.gpr(); GPRReg reg2 = op2.gpr(); shiftOp(op, reg1, reg2, result.gpr()); - integerResult(result.gpr(), node); + int32Result(result.gpr(), node); } break; @@ -2155,16 +2125,91 @@ void SpeculativeJIT::compile(Node* node) break; } - case Int32ToDouble: - case ForwardInt32ToDouble: { - compileInt32ToDouble(node); + case DoubleRep: { + compileDoubleRep(node); + break; + } + + case ValueRep: { + compileValueRep(node); + break; + } + + case Int52Rep: { + switch (node->child1().useKind()) { + case Int32Use: { + SpeculateInt32Operand operand(this, node->child1()); + GPRTemporary result(this, Reuse, operand); + + m_jit.signExtend32ToPtr(operand.gpr(), result.gpr()); + + strictInt52Result(result.gpr(), node); + break; + } + + case MachineIntUse: { + GPRTemporary result(this); + GPRReg resultGPR = result.gpr(); + + convertMachineInt(node->child1(), resultGPR); + + strictInt52Result(resultGPR, node); + break; + } + + case DoubleRepMachineIntUse: { + SpeculateDoubleOperand value(this, node->child1()); + FPRReg valueFPR = value.fpr(); + + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + + callOperation(operationConvertDoubleToInt52, resultGPR, valueFPR); + + DFG_TYPE_CHECK( + JSValueRegs(), node->child1(), SpecInt52AsDouble, + m_jit.branch64( + JITCompiler::Equal, resultGPR, + JITCompiler::TrustedImm64(JSValue::notInt52))); + + strictInt52Result(resultGPR, node); + break; + } + + default: + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); + } + break; + } + + case ValueAdd: { + JSValueOperand op1(this, node->child1()); + JSValueOperand op2(this, node->child2()); + + GPRReg op1GPR = op1.gpr(); + GPRReg op2GPR = op2.gpr(); + + flushRegisters(); + + GPRFlushedCallResult result(this); + if (isKnownNotNumber(node->child1().node()) || isKnownNotNumber(node->child2().node())) + callOperation(operationValueAddNotNumber, result.gpr(), op1GPR, op2GPR); + else + callOperation(operationValueAdd, result.gpr(), op1GPR, op2GPR); + + jsValueResult(result.gpr(), node); break; } - case ValueAdd: case ArithAdd: compileAdd(node); break; + + case ArithClz32: + compileArithClz32(node); + break; case MakeRope: compileMakeRope(node); @@ -2182,41 +2227,8 @@ void SpeculativeJIT::compile(Node* node) compileArithMul(node); break; - case ArithIMul: - compileArithIMul(node); - break; - case ArithDiv: { - switch (node->binaryUseKind()) { - case Int32Use: { -#if CPU(X86) || CPU(X86_64) - compileIntegerArithDivForX86(node); -#elif CPU(ARM64) - compileIntegerArithDivForARM64(node); -#else - // See DFGFixupPhase - on any architecture other than X86[_64] we'll force the prediction to double. - ASSERT_NOT_REACHED(); -#endif - break; - } - - case NumberUse: { - SpeculateDoubleOperand op1(this, node->child1()); - SpeculateDoubleOperand op2(this, node->child2()); - FPRTemporary result(this, op1); - - FPRReg reg1 = op1.fpr(); - FPRReg reg2 = op2.fpr(); - m_jit.divDouble(reg1, reg2, result.fpr()); - - doubleResult(result.fpr(), node); - break; - } - - default: - RELEASE_ASSERT_NOT_REACHED(); - break; - } + compileArithDiv(node); break; } @@ -2228,20 +2240,20 @@ void SpeculativeJIT::compile(Node* node) case ArithAbs: { switch (node->child1().useKind()) { case Int32Use: { - SpeculateIntegerOperand op1(this, node->child1()); + SpeculateStrictInt32Operand op1(this, node->child1()); GPRTemporary result(this); GPRTemporary scratch(this); - m_jit.zeroExtend32ToPtr(op1.gpr(), result.gpr()); + m_jit.move(op1.gpr(), result.gpr()); m_jit.rshift32(result.gpr(), MacroAssembler::TrustedImm32(31), scratch.gpr()); m_jit.add32(scratch.gpr(), result.gpr()); m_jit.xor32(scratch.gpr(), result.gpr()); speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, result.gpr(), MacroAssembler::TrustedImm32(1 << 31))); - integerResult(result.gpr(), node); + int32Result(result.gpr(), node); break; } - case NumberUse: { + case DoubleRepUse: { SpeculateDoubleOperand op1(this, node->child1()); FPRTemporary result(this); @@ -2251,7 +2263,7 @@ void SpeculativeJIT::compile(Node* node) } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } break; @@ -2263,7 +2275,7 @@ void SpeculativeJIT::compile(Node* node) case Int32Use: { SpeculateStrictInt32Operand op1(this, node->child1()); SpeculateStrictInt32Operand op2(this, node->child2()); - GPRTemporary result(this, op1); + GPRTemporary result(this, Reuse, op1); MacroAssembler::Jump op1Less = m_jit.branch32(op == ArithMin ? MacroAssembler::LessThan : MacroAssembler::GreaterThan, op1.gpr(), op2.gpr()); m_jit.move(op2.gpr(), result.gpr()); @@ -2275,11 +2287,11 @@ void SpeculativeJIT::compile(Node* node) } else op1Less.link(&m_jit); - integerResult(result.gpr(), node); + int32Result(result.gpr(), node); break; } - case NumberUse: { + case DoubleRepUse: { SpeculateDoubleOperand op1(this, node->child1()); SpeculateDoubleOperand op2(this, node->child2()); FPRTemporary result(this, op1); @@ -2318,22 +2330,63 @@ void SpeculativeJIT::compile(Node* node) } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } break; } - - case ArithSqrt: { + + case ArithPow: + compileArithPow(node); + break; + + case ArithSqrt: + compileArithSqrt(node); + break; + + case ArithFRound: { SpeculateDoubleOperand op1(this, node->child1()); FPRTemporary result(this, op1); - m_jit.sqrtDouble(op1.fpr(), result.fpr()); + m_jit.convertDoubleToFloat(op1.fpr(), result.fpr()); + m_jit.convertFloatToDouble(result.fpr(), result.fpr()); + + doubleResult(result.fpr(), node); + break; + } + + case ArithRound: + compileArithRound(node); + break; + + case ArithSin: { + SpeculateDoubleOperand op1(this, node->child1()); + FPRReg op1FPR = op1.fpr(); + + flushRegisters(); + + FPRResult result(this); + callOperation(sin, result.fpr(), op1FPR); + doubleResult(result.fpr(), node); + break; + } + + case ArithCos: { + SpeculateDoubleOperand op1(this, node->child1()); + FPRReg op1FPR = op1.fpr(); + + flushRegisters(); + FPRResult result(this); + callOperation(cos, result.fpr(), op1FPR); doubleResult(result.fpr(), node); break; } + case ArithLog: + compileArithLog(node); + break; + case LogicalNot: compileLogicalNot(node); break; @@ -2359,7 +2412,7 @@ void SpeculativeJIT::compile(Node* node) break; case CompareEqConstant: - ASSERT(isNullConstant(node->child2().node())); + ASSERT(node->child2()->asJSValue().isNull()); if (nonSpeculativeCompareNull(node, node->child1())) return; break; @@ -2369,11 +2422,6 @@ void SpeculativeJIT::compile(Node* node) return; break; - case CompareStrictEqConstant: - if (compileStrictEqForConstant(node, node->child1(), valueOfJSConstant(node->child2().node()))) - return; - break; - case CompareStrictEq: if (compileStrictEq(node)) return; @@ -2410,8 +2458,7 @@ void SpeculativeJIT::compile(Node* node) switch (node->arrayMode().type()) { case Array::SelectUsingPredictions: case Array::ForceExit: - RELEASE_ASSERT_NOT_REACHED(); - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); + DFG_CRASH(m_jit.graph(), node, "Bad array mode type"); break; case Array::Generic: { JSValueOperand base(this, node->child1()); @@ -2420,7 +2467,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg propertyGPR = property.gpr(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation(operationGetByVal, result.gpr(), baseGPR, propertyGPR); jsValueResult(result.gpr(), node); @@ -2442,8 +2489,18 @@ void SpeculativeJIT::compile(Node* node) GPRTemporary result(this); m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), result.gpr()); - speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchTest64(MacroAssembler::Zero, result.gpr())); - jsValueResult(result.gpr(), node, node->arrayMode().type() == Array::Int32 ? DataFormatJSInteger : DataFormatJS); + if (node->arrayMode().isSaneChain()) { + ASSERT(node->arrayMode().type() == Array::Contiguous); + JITCompiler::Jump notHole = m_jit.branchTest64( + MacroAssembler::NonZero, result.gpr()); + m_jit.move(TrustedImm64(JSValue::encode(jsUndefined())), result.gpr()); + notHole.link(&m_jit); + } else { + speculationCheck( + LoadFromHole, JSValueRegs(), 0, + m_jit.branchTest64(MacroAssembler::Zero, result.gpr())); + } + jsValueResult(result.gpr(), node, node->arrayMode().type() == Array::Int32 ? DataFormatJSInt32 : DataFormatJS); break; } @@ -2479,13 +2536,6 @@ void SpeculativeJIT::compile(Node* node) case Array::Double: { if (node->arrayMode().isInBounds()) { - if (node->arrayMode().isSaneChain()) { - JSGlobalObject* globalObject = m_jit.globalObjectFor(node->codeOrigin); - ASSERT(globalObject->arrayPrototypeChainIsSane()); - globalObject->arrayPrototype()->structure()->addTransitionWatchpoint(speculationWatchpoint()); - globalObject->objectPrototype()->structure()->addTransitionWatchpoint(speculationWatchpoint()); - } - SpeculateStrictInt32Operand property(this, node->child2()); StorageOperand storage(this, node->child3()); @@ -2592,43 +2642,23 @@ void SpeculativeJIT::compile(Node* node) case Array::String: compileGetByValOnString(node); break; - case Array::Arguments: - compileGetByValOnArguments(node); - break; - case Array::Int8Array: - compileGetByValOnIntTypedArray(m_jit.vm()->int8ArrayDescriptor(), node, sizeof(int8_t), SignedTypedArray); - break; - case Array::Int16Array: - compileGetByValOnIntTypedArray(m_jit.vm()->int16ArrayDescriptor(), node, sizeof(int16_t), SignedTypedArray); + case Array::DirectArguments: + compileGetByValOnDirectArguments(node); break; - case Array::Int32Array: - compileGetByValOnIntTypedArray(m_jit.vm()->int32ArrayDescriptor(), node, sizeof(int32_t), SignedTypedArray); - break; - case Array::Uint8Array: - compileGetByValOnIntTypedArray(m_jit.vm()->uint8ArrayDescriptor(), node, sizeof(uint8_t), UnsignedTypedArray); - break; - case Array::Uint8ClampedArray: - compileGetByValOnIntTypedArray(m_jit.vm()->uint8ClampedArrayDescriptor(), node, sizeof(uint8_t), UnsignedTypedArray); - break; - case Array::Uint16Array: - compileGetByValOnIntTypedArray(m_jit.vm()->uint16ArrayDescriptor(), node, sizeof(uint16_t), UnsignedTypedArray); - break; - case Array::Uint32Array: - compileGetByValOnIntTypedArray(m_jit.vm()->uint32ArrayDescriptor(), node, sizeof(uint32_t), UnsignedTypedArray); - break; - case Array::Float32Array: - compileGetByValOnFloatTypedArray(m_jit.vm()->float32ArrayDescriptor(), node, sizeof(float)); - break; - case Array::Float64Array: - compileGetByValOnFloatTypedArray(m_jit.vm()->float64ArrayDescriptor(), node, sizeof(double)); - break; - default: - RELEASE_ASSERT_NOT_REACHED(); + case Array::ScopedArguments: + compileGetByValOnScopedArguments(node); break; - } + default: { + TypedArrayType type = node->arrayMode().typedArrayType(); + if (isInt(type)) + compileGetByValOnIntTypedArray(node, type); + else + compileGetByValOnFloatTypedArray(node, type); + } } break; } + case PutByValDirect: case PutByVal: case PutByValAlias: { Edge child1 = m_jit.graph().varArgChild(node, 0); @@ -2642,12 +2672,10 @@ void SpeculativeJIT::compile(Node* node) switch (arrayMode.type()) { case Array::SelectUsingPredictions: case Array::ForceExit: - RELEASE_ASSERT_NOT_REACHED(); - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); - alreadyHandled = true; + DFG_CRASH(m_jit.graph(), node, "Bad array mode type"); break; case Array::Generic: { - RELEASE_ASSERT(node->op() == PutByVal); + DFG_ASSERT(m_jit.graph(), node, node->op() == PutByVal || node->op() == PutByValDirect); JSValueOperand arg1(this, child1); JSValueOperand arg2(this, child2); @@ -2656,8 +2684,10 @@ void SpeculativeJIT::compile(Node* node) GPRReg arg2GPR = arg2.gpr(); GPRReg arg3GPR = arg3.gpr(); flushRegisters(); - - callOperation(m_jit.strictModeFor(node->codeOrigin) ? operationPutByValStrict : operationPutByValNonStrict, arg1GPR, arg2GPR, arg3GPR); + if (node->op() == PutByValDirect) + callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict, arg1GPR, arg2GPR, arg3GPR); + else + callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict, arg1GPR, arg2GPR, arg3GPR); noResult(node); alreadyHandled = true; @@ -2695,11 +2725,6 @@ void SpeculativeJIT::compile(Node* node) m_jit.branch64( MacroAssembler::Below, valueReg, GPRInfo::tagTypeNumberRegister)); } - - if (arrayMode.type() == Array::Contiguous && Heap::isWriteBarrierEnabled()) { - GPRTemporary scratch(this); - writeBarrier(baseReg, value.gpr(), child3, WriteBarrierForPropertyAccess, scratch.gpr()); - } StorageOperand storage(this, child4); GPRReg storageReg = storage.gpr(); @@ -2721,7 +2746,7 @@ void SpeculativeJIT::compile(Node* node) if (arrayMode.isInBounds()) { speculationCheck( - StoreToHoleOrOutOfBounds, JSValueRegs(), 0, + OutOfBounds, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::AboveOrEqual, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength()))); } else { MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, propertyReg, MacroAssembler::Address(storageReg, Butterfly::offsetOfPublicLength())); @@ -2745,11 +2770,17 @@ void SpeculativeJIT::compile(Node* node) storage.use(); if (arrayMode.isOutOfBounds()) { - addSlowPathGenerator( - slowPathCall( + if (node->op() == PutByValDirect) { + addSlowPathGenerator(slowPathCall( + slowCase, this, + m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectBeyondArrayBoundsStrict : operationPutByValDirectBeyondArrayBoundsNonStrict, + NoResult, baseReg, propertyReg, valueReg)); + } else { + addSlowPathGenerator(slowPathCall( slowCase, this, m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict, NoResult, baseReg, propertyReg, valueReg)); + } } noResult(node, UseChildrenCalledExplicitly); @@ -2769,11 +2800,6 @@ void SpeculativeJIT::compile(Node* node) if (!m_compileOkay) return; - - if (Heap::isWriteBarrierEnabled()) { - GPRTemporary scratch(this); - writeBarrier(baseReg, value.gpr(), child3, WriteBarrierForPropertyAccess, scratch.gpr()); - } StorageOperand storage(this, child4); GPRReg storageReg = storage.gpr(); @@ -2833,99 +2859,30 @@ void SpeculativeJIT::compile(Node* node) storage.use(); if (!slowCases.empty()) { - addSlowPathGenerator( - slowPathCall( + if (node->op() == PutByValDirect) { + addSlowPathGenerator(slowPathCall( + slowCases, this, + m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectBeyondArrayBoundsStrict : operationPutByValDirectBeyondArrayBoundsNonStrict, + NoResult, baseReg, propertyReg, valueReg)); + } else { + addSlowPathGenerator(slowPathCall( slowCases, this, m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict, NoResult, baseReg, propertyReg, valueReg)); + } } noResult(node, UseChildrenCalledExplicitly); break; } - case Array::Arguments: { - JSValueOperand value(this, child3); - GPRTemporary scratch(this); - GPRTemporary scratch2(this); - - GPRReg valueReg = value.gpr(); - GPRReg scratchReg = scratch.gpr(); - GPRReg scratch2Reg = scratch2.gpr(); - - if (!m_compileOkay) - return; - - // Two really lame checks. - speculationCheck( - Uncountable, JSValueSource(), 0, - m_jit.branch32( - MacroAssembler::AboveOrEqual, propertyReg, - MacroAssembler::Address(baseReg, OBJECT_OFFSETOF(Arguments, m_numArguments)))); - speculationCheck( - Uncountable, JSValueSource(), 0, - m_jit.branchTestPtr( - MacroAssembler::NonZero, - MacroAssembler::Address( - baseReg, OBJECT_OFFSETOF(Arguments, m_slowArguments)))); - - m_jit.move(propertyReg, scratch2Reg); - m_jit.neg32(scratch2Reg); - m_jit.signExtend32ToPtr(scratch2Reg, scratch2Reg); - m_jit.loadPtr( - MacroAssembler::Address(baseReg, OBJECT_OFFSETOF(Arguments, m_registers)), - scratchReg); - - m_jit.store64( - valueReg, - MacroAssembler::BaseIndex( - scratchReg, scratch2Reg, MacroAssembler::TimesEight, - CallFrame::thisArgumentOffset() * sizeof(Register) - sizeof(Register))); - - noResult(node); - break; - } - - case Array::Int8Array: - compilePutByValForIntTypedArray(m_jit.vm()->int8ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int8_t), SignedTypedArray); - break; - - case Array::Int16Array: - compilePutByValForIntTypedArray(m_jit.vm()->int16ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int16_t), SignedTypedArray); - break; - - case Array::Int32Array: - compilePutByValForIntTypedArray(m_jit.vm()->int32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(int32_t), SignedTypedArray); - break; - - case Array::Uint8Array: - compilePutByValForIntTypedArray(m_jit.vm()->uint8ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint8_t), UnsignedTypedArray); - break; - - case Array::Uint8ClampedArray: - compilePutByValForIntTypedArray(m_jit.vm()->uint8ClampedArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint8_t), UnsignedTypedArray, ClampRounding); - break; - - case Array::Uint16Array: - compilePutByValForIntTypedArray(m_jit.vm()->uint16ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint16_t), UnsignedTypedArray); - break; - - case Array::Uint32Array: - compilePutByValForIntTypedArray(m_jit.vm()->uint32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(uint32_t), UnsignedTypedArray); - break; - - case Array::Float32Array: - compilePutByValForFloatTypedArray(m_jit.vm()->float32ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(float)); - break; - - case Array::Float64Array: - compilePutByValForFloatTypedArray(m_jit.vm()->float64ArrayDescriptor(), base.gpr(), property.gpr(), node, sizeof(double)); - break; - - default: - RELEASE_ASSERT_NOT_REACHED(); - break; - } + default: { + TypedArrayType type = arrayMode.typedArrayType(); + if (isInt(type)) + compilePutByValForIntTypedArray(base.gpr(), property.gpr(), node, type); + else + compilePutByValForFloatTypedArray(base.gpr(), property.gpr(), node, type); + } } break; } @@ -2940,7 +2897,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg argumentGPR = argument.gpr(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR); // Must use jsValueResult because otherwise we screw up register @@ -2955,7 +2912,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg argumentGPR = argument.gpr(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation(operationRegExpExec, result.gpr(), baseGPR, argumentGPR); jsValueResult(result.gpr(), node); @@ -2969,7 +2926,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg argumentGPR = argument.gpr(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR); // If we add a DataFormatBool, we should use it here. @@ -3003,11 +2960,6 @@ void SpeculativeJIT::compile(Node* node) MacroAssembler::Below, valueGPR, GPRInfo::tagTypeNumberRegister)); } - if (node->arrayMode().type() != Array::Int32 && Heap::isWriteBarrierEnabled()) { - GPRTemporary scratch(this); - writeBarrier(baseGPR, valueGPR, node->child2(), WriteBarrierForPropertyAccess, scratch.gpr(), storageLengthGPR); - } - m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR); MacroAssembler::Jump slowPath = m_jit.branch32(MacroAssembler::AboveOrEqual, storageLengthGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength())); m_jit.store64(valueGPR, MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight)); @@ -3017,7 +2969,7 @@ void SpeculativeJIT::compile(Node* node) addSlowPathGenerator( slowPathCall( - slowPath, this, operationArrayPush, NoResult, storageLengthGPR, + slowPath, this, operationArrayPush, storageLengthGPR, valueGPR, baseGPR)); jsValueResult(storageLengthGPR, node); @@ -3029,7 +2981,7 @@ void SpeculativeJIT::compile(Node* node) FPRReg valueFPR = value.fpr(); DFG_TYPE_CHECK( - JSValueRegs(), node->child2(), SpecRealNumber, + JSValueRegs(), node->child2(), SpecDoubleReal, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, valueFPR, valueFPR)); m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR); @@ -3041,7 +2993,7 @@ void SpeculativeJIT::compile(Node* node) addSlowPathGenerator( slowPathCall( - slowPath, this, operationArrayPushDouble, NoResult, storageLengthGPR, + slowPath, this, operationArrayPushDouble, storageLengthGPR, valueFPR, baseGPR)); jsValueResult(storageLengthGPR, node); @@ -3052,11 +3004,6 @@ void SpeculativeJIT::compile(Node* node) JSValueOperand value(this, node->child2()); GPRReg valueGPR = value.gpr(); - if (Heap::isWriteBarrierEnabled()) { - GPRTemporary scratch(this); - writeBarrier(baseGPR, valueGPR, node->child2(), WriteBarrierForPropertyAccess, scratch.gpr(), storageLengthGPR); - } - m_jit.load32(MacroAssembler::Address(storageGPR, ArrayStorage::lengthOffset()), storageLengthGPR); // Refuse to handle bizarre lengths. @@ -3121,7 +3068,7 @@ void SpeculativeJIT::compile(Node* node) // FIXME: This would not have to be here if changing the publicLength also zeroed the values between the old // length and the new length. m_jit.store64( - MacroAssembler::TrustedImm64((int64_t)0), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight)); + MacroAssembler::TrustedImm64(bitwise_cast(PNaN)), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight)); slowCase = m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempFPR, tempFPR); boxDouble(tempFPR, valueGPR); } else { @@ -3188,8 +3135,7 @@ void SpeculativeJIT::compile(Node* node) } case DFG::Jump: { - BlockIndex taken = node->takenBlockIndex(); - jump(taken); + jump(node->targetBlock()); noResult(node); break; } @@ -3197,27 +3143,21 @@ void SpeculativeJIT::compile(Node* node) case Branch: emitBranch(node); break; + + case Switch: + emitSwitch(node); + break; case Return: { ASSERT(GPRInfo::callFrameRegister != GPRInfo::regT1); ASSERT(GPRInfo::regT1 != GPRInfo::returnValueGPR); ASSERT(GPRInfo::returnValueGPR != GPRInfo::callFrameRegister); -#if DFG_ENABLE(SUCCESS_STATS) - static SamplingCounter counter("SpeculativeJIT"); - m_jit.emitCount(counter); -#endif - // Return the result in returnValueGPR. JSValueOperand op1(this, node->child1()); m_jit.move(op1.gpr(), GPRInfo::returnValueGPR); - // Grab the return address. - m_jit.emitGetFromCallFrameHeaderPtr(JSStack::ReturnPC, GPRInfo::regT1); - // Restore our caller's "r". - m_jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, GPRInfo::callFrameRegister); - // Return. - m_jit.restoreReturnAddressBeforeReturn(GPRInfo::regT1); + m_jit.emitFunctionEpilogue(); m_jit.ret(); noResult(node); @@ -3232,64 +3172,110 @@ void SpeculativeJIT::compile(Node* node) break; } - case ToPrimitive: { - RELEASE_ASSERT(node->child1().useKind() == UntypedUse); - JSValueOperand op1(this, node->child1()); - GPRTemporary result(this, op1); - - GPRReg op1GPR = op1.gpr(); - GPRReg resultGPR = result.gpr(); - - op1.use(); - - if (!(m_state.forNode(node->child1()).m_type & ~(SpecNumber | SpecBoolean))) - m_jit.move(op1GPR, resultGPR); - else { - MacroAssembler::Jump alreadyPrimitive = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagMaskRegister); - MacroAssembler::Jump notPrimitive = m_jit.branchPtr(MacroAssembler::NotEqual, MacroAssembler::Address(op1GPR, JSCell::structureOffset()), MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())); - - alreadyPrimitive.link(&m_jit); - m_jit.move(op1GPR, resultGPR); + case BooleanToNumber: { + switch (node->child1().useKind()) { + case BooleanUse: { + JSValueOperand value(this, node->child1(), ManualOperandSpeculation); + GPRTemporary result(this); // FIXME: We could reuse, but on speculation fail would need recovery to restore tag (akin to add). - addSlowPathGenerator( - slowPathCall(notPrimitive, this, operationToPrimitive, resultGPR, op1GPR)); + m_jit.move(value.gpr(), result.gpr()); + m_jit.xor64(TrustedImm32(static_cast(ValueFalse)), result.gpr()); + DFG_TYPE_CHECK( + JSValueRegs(value.gpr()), node->child1(), SpecBoolean, m_jit.branchTest64( + JITCompiler::NonZero, result.gpr(), TrustedImm32(static_cast(~1)))); + + int32Result(result.gpr(), node); + break; } - - jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); - break; - } - - case ToString: { - if (node->child1().useKind() == UntypedUse) { - JSValueOperand op1(this, node->child1()); - GPRReg op1GPR = op1.gpr(); - GPRResult result(this); - GPRReg resultGPR = result.gpr(); + case UntypedUse: { + JSValueOperand value(this, node->child1()); + GPRTemporary result(this); + + if (!m_interpreter.needsTypeCheck(node->child1(), SpecBoolInt32 | SpecBoolean)) { + m_jit.move(value.gpr(), result.gpr()); + m_jit.and32(TrustedImm32(1), result.gpr()); + int32Result(result.gpr(), node); + break; + } + + m_jit.move(value.gpr(), result.gpr()); + m_jit.xor64(TrustedImm32(static_cast(ValueFalse)), result.gpr()); + JITCompiler::Jump isBoolean = m_jit.branchTest64( + JITCompiler::Zero, result.gpr(), TrustedImm32(static_cast(~1))); + m_jit.move(value.gpr(), result.gpr()); + JITCompiler::Jump done = m_jit.jump(); + isBoolean.link(&m_jit); + m_jit.or64(GPRInfo::tagTypeNumberRegister, result.gpr()); + done.link(&m_jit); + + jsValueResult(result.gpr(), node); + break; + } + + default: + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); + break; + } + break; + } + + case ToPrimitive: { + DFG_ASSERT(m_jit.graph(), node, node->child1().useKind() == UntypedUse); + JSValueOperand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); + + GPRReg op1GPR = op1.gpr(); + GPRReg resultGPR = result.gpr(); + + op1.use(); + + MacroAssembler::Jump alreadyPrimitive = m_jit.branchIfNotCell(JSValueRegs(op1GPR)); + MacroAssembler::Jump notPrimitive = m_jit.branchIfObject(op1GPR); + + alreadyPrimitive.link(&m_jit); + m_jit.move(op1GPR, resultGPR); + + addSlowPathGenerator( + slowPathCall(notPrimitive, this, operationToPrimitive, resultGPR, op1GPR)); + + jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); + break; + } + + case ToString: + case CallStringConstructor: { + if (node->child1().useKind() == UntypedUse) { + JSValueOperand op1(this, node->child1()); + GPRReg op1GPR = op1.gpr(); + + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); flushRegisters(); JITCompiler::Jump done; if (node->child1()->prediction() & SpecString) { - JITCompiler::Jump slowPath1 = m_jit.branchTest64( - JITCompiler::NonZero, op1GPR, GPRInfo::tagMaskRegister); - JITCompiler::Jump slowPath2 = m_jit.branchPtr( - JITCompiler::NotEqual, - JITCompiler::Address(op1GPR, JSCell::structureOffset()), - TrustedImmPtr(m_jit.vm()->stringStructure.get())); + JITCompiler::Jump slowPath1 = m_jit.branchIfNotCell(JSValueRegs(op1GPR)); + JITCompiler::Jump slowPath2 = m_jit.branchIfNotString(op1GPR); m_jit.move(op1GPR, resultGPR); done = m_jit.jump(); slowPath1.link(&m_jit); slowPath2.link(&m_jit); } - callOperation(operationToString, resultGPR, op1GPR); + if (op == ToString) + callOperation(operationToString, resultGPR, op1GPR); + else { + ASSERT(op == CallStringConstructor); + callOperation(operationCallStringConstructor, resultGPR, op1GPR); + } if (done.isSet()) done.link(&m_jit); cellResult(resultGPR, node); break; } - compileToStringOnCell(node); + compileToStringOrCallStringConstructorOnCell(node); break; } @@ -3299,12 +3285,10 @@ void SpeculativeJIT::compile(Node* node) } case NewArray: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) { - globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint()); - + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) { Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()); - RELEASE_ASSERT(structure->indexingType() == node->indexingType()); + DFG_ASSERT(m_jit.graph(), node, structure->indexingType() == node->indexingType()); ASSERT( hasUndecided(structure->indexingType()) || hasInt32(structure->indexingType()) @@ -3337,7 +3321,7 @@ void SpeculativeJIT::compile(Node* node) SpeculateDoubleOperand operand(this, use); FPRReg opFPR = operand.fpr(); DFG_TYPE_CHECK( - JSValueRegs(), use, SpecRealNumber, + JSValueRegs(), use, SpecDoubleReal, m_jit.branchDouble( MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR)); m_jit.storeDouble(opFPR, MacroAssembler::Address(storageGPR, sizeof(double) * operandIdx)); @@ -3375,7 +3359,7 @@ void SpeculativeJIT::compile(Node* node) if (!node->numChildren()) { flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation(operationNewEmptyArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())); cellResult(result.gpr(), node); break; @@ -3402,7 +3386,7 @@ void SpeculativeJIT::compile(Node* node) FPRReg opFPR = operand.fpr(); GPRReg scratchGPR = scratch.gpr(); DFG_TYPE_CHECK( - JSValueRegs(), use, SpecRealNumber, + JSValueRegs(), use, SpecDoubleReal, m_jit.branchDouble( MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR)); m_jit.boxDouble(opFPR, scratchGPR); @@ -3454,7 +3438,7 @@ void SpeculativeJIT::compile(Node* node) m_jit.storePtr(TrustedImmPtr(scratchSize), scratch.gpr()); } - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation( operationNewArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), @@ -3472,10 +3456,8 @@ void SpeculativeJIT::compile(Node* node) } case NewArrayWithSize: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) { - globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint()); - + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) { SpeculateStrictInt32Operand size(this, node->child1()); GPRTemporary result(this); GPRTemporary storage(this); @@ -3489,7 +3471,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg scratch2GPR = scratch2.gpr(); MacroAssembler::JumpList slowCases; - slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX))); + slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH))); ASSERT((1 << 3) == sizeof(JSValue)); m_jit.move(sizeGPR, scratchGPR); @@ -3499,13 +3481,13 @@ void SpeculativeJIT::compile(Node* node) emitAllocateBasicStorage(resultGPR, storageGPR)); m_jit.subPtr(scratchGPR, storageGPR); Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()); - emitAllocateJSObject(resultGPR, ImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases); + emitAllocateJSObject(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases); m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength())); m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength())); if (hasDouble(node->indexingType())) { - m_jit.move(TrustedImm64(bitwise_cast(QNaN)), scratchGPR); + m_jit.move(TrustedImm64(bitwise_cast(PNaN)), scratchGPR); m_jit.move(sizeGPR, scratch2GPR); MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratch2GPR); MacroAssembler::Label loop = m_jit.label(); @@ -3515,12 +3497,11 @@ void SpeculativeJIT::compile(Node* node) done.link(&m_jit); } - addSlowPathGenerator(adoptPtr( - new CallArrayAllocatorWithVariableSizeSlowPathGenerator( + addSlowPathGenerator(std::make_unique( slowCases, this, operationNewArrayWithSize, resultGPR, globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage), - sizeGPR))); + sizeGPR)); cellResult(resultGPR, node); break; @@ -3529,10 +3510,10 @@ void SpeculativeJIT::compile(Node* node) SpeculateStrictInt32Operand size(this, node->child1()); GPRReg sizeGPR = size.gpr(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); GPRReg structureGPR = selectScratchGPR(sizeGPR); - MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX)); + MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)); m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), structureGPR); MacroAssembler::Jump done = m_jit.jump(); bigLength.link(&m_jit); @@ -3544,11 +3525,9 @@ void SpeculativeJIT::compile(Node* node) } case NewArrayBuffer: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); IndexingType indexingType = node->indexingType(); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(indexingType)) { - globalObject->havingABadTimeWatchpoint()->add(speculationWatchpoint()); - + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(indexingType)) { unsigned numElements = node->numConstants(); GPRTemporary result(this); @@ -3559,7 +3538,7 @@ void SpeculativeJIT::compile(Node* node) emitAllocateJSArray(resultGPR, globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType), storageGPR, numElements); - RELEASE_ASSERT(indexingType & IsArray); + DFG_ASSERT(m_jit.graph(), node, indexingType & IsArray); JSValue* data = m_jit.codeBlock()->constantBuffer(node->startConstant()); if (indexingType == ArrayWithDouble) { for (unsigned index = 0; index < node->numConstants(); ++index) { @@ -3581,7 +3560,7 @@ void SpeculativeJIT::compile(Node* node) } flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation(operationNewArrayBuffer, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), node->startConstant(), node->numConstants()); @@ -3589,9 +3568,39 @@ void SpeculativeJIT::compile(Node* node) break; } + case NewTypedArray: { + switch (node->child1().useKind()) { + case Int32Use: + compileNewTypedArray(node); + break; + case UntypedUse: { + JSValueOperand argument(this, node->child1()); + GPRReg argumentGPR = argument.gpr(); + + flushRegisters(); + + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + callOperation( + operationNewTypedArrayWithOneArgumentForType(node->typedArrayType()), + resultGPR, globalObject->typedArrayStructure(node->typedArrayType()), + argumentGPR); + + cellResult(resultGPR, node); + break; + } + default: + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); + break; + } + break; + } + case NewRegexp: { flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation(operationNewRegexp, result.gpr(), m_jit.codeBlock()->regexp(node->regexpIndex())); @@ -3599,17 +3608,29 @@ void SpeculativeJIT::compile(Node* node) break; } - case ConvertThis: { + case ToThis: { ASSERT(node->child1().useKind() == UntypedUse); JSValueOperand thisValue(this, node->child1()); + GPRTemporary temp(this); GPRReg thisValueGPR = thisValue.gpr(); + GPRReg tempGPR = temp.gpr(); - flushRegisters(); - - GPRResult result(this); - callOperation(operationConvertThis, result.gpr(), thisValueGPR); - - cellResult(result.gpr(), node); + MacroAssembler::JumpList slowCases; + slowCases.append(m_jit.branchIfNotCell(JSValueRegs(thisValueGPR))); + slowCases.append(m_jit.branch8( + MacroAssembler::NotEqual, + MacroAssembler::Address(thisValueGPR, JSCell::typeInfoTypeOffset()), + TrustedImm32(FinalObjectType))); + m_jit.move(thisValueGPR, tempGPR); + J_JITOperation_EJ function; + if (m_jit.graph().executableFor(node->origin.semantic)->isStrictMode()) + function = operationToThisStrict; + else + function = operationToThis; + addSlowPathGenerator( + slowPathCall(slowCases, this, function, tempGPR, thisValueGPR)); + + jsValueResult(tempGPR, node); break; } @@ -3631,11 +3652,16 @@ void SpeculativeJIT::compile(Node* node) GPRReg allocatorGPR = allocator.gpr(); GPRReg structureGPR = structure.gpr(); GPRReg scratchGPR = scratch.gpr(); + // Rare data is only used to access the allocator & structure + // We can avoid using an additional GPR this way + GPRReg rareDataGPR = structureGPR; MacroAssembler::JumpList slowPath; - m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR); - m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR); + m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfRareData()), rareDataGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, rareDataGPR)); + m_jit.loadPtr(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR); + m_jit.loadPtr(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR); slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, allocatorGPR)); emitAllocateJSObject(resultGPR, allocatorGPR, structureGPR, TrustedImmPtr(0), scratchGPR, slowPath); @@ -3645,12 +3671,6 @@ void SpeculativeJIT::compile(Node* node) break; } - case AllocationProfileWatchpoint: { - jsCast(node->function())->addAllocationProfileWatchpoint(speculationWatchpoint()); - noResult(node); - break; - } - case NewObject: { GPRTemporary result(this); GPRTemporary allocator(this); @@ -3663,7 +3683,7 @@ void SpeculativeJIT::compile(Node* node) MacroAssembler::JumpList slowPath; Structure* structure = node->structure(); - size_t allocationSize = JSObject::allocationSize(structure->inlineCapacity()); + size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity()); MarkedAllocator* allocatorPtr = &m_jit.vm()->heap.allocatorForObjectWithoutDestructor(allocationSize); m_jit.move(TrustedImmPtr(allocatorPtr), allocatorGPR); @@ -3677,119 +3697,61 @@ void SpeculativeJIT::compile(Node* node) case GetCallee: { GPRTemporary result(this); - m_jit.loadPtr(JITCompiler::addressFor(static_cast(node->codeOrigin.stackOffset() + static_cast(JSStack::Callee))), result.gpr()); - cellResult(result.gpr(), node); - break; - } - - case SetCallee: { - SpeculateCellOperand callee(this, node->child1()); - m_jit.storePtr(callee.gpr(), JITCompiler::addressFor(static_cast(node->codeOrigin.stackOffset() + static_cast(JSStack::Callee)))); - noResult(node); - break; - } - - case GetScope: { - SpeculateCellOperand function(this, node->child1()); - GPRTemporary result(this, function); - m_jit.loadPtr(JITCompiler::Address(function.gpr(), JSFunction::offsetOfScopeChain()), result.gpr()); + m_jit.loadPtr(JITCompiler::addressFor(JSStack::Callee), result.gpr()); cellResult(result.gpr(), node); break; } - case GetMyScope: { + case GetArgumentCount: { GPRTemporary result(this); - GPRReg resultGPR = result.gpr(); - - m_jit.loadPtr(JITCompiler::addressFor(static_cast(node->codeOrigin.stackOffset() + static_cast(JSStack::ScopeChain))), resultGPR); - cellResult(resultGPR, node); - break; - } - - case SetMyScope: { - SpeculateCellOperand callee(this, node->child1()); - m_jit.storePtr(callee.gpr(), JITCompiler::addressFor(static_cast(node->codeOrigin.stackOffset() + static_cast(JSStack::ScopeChain)))); - noResult(node); + m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), result.gpr()); + int32Result(result.gpr(), node); break; } - case SkipTopScope: { - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this, scope); - GPRReg resultGPR = result.gpr(); - m_jit.move(scope.gpr(), resultGPR); - JITCompiler::Jump activationNotCreated = - m_jit.branchTest64( - JITCompiler::Zero, - JITCompiler::addressFor( - static_cast(m_jit.codeBlock()->activationRegister()))); - m_jit.loadPtr(JITCompiler::Address(resultGPR, JSScope::offsetOfNext()), resultGPR); - activationNotCreated.link(&m_jit); - cellResult(resultGPR, node); + case GetScope: + compileGetScope(node); break; - } - case SkipScope: { - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this, scope); - m_jit.loadPtr(JITCompiler::Address(scope.gpr(), JSScope::offsetOfNext()), result.gpr()); - cellResult(result.gpr(), node); + case SkipScope: + compileSkipScope(node); break; - } - case GetScopeRegisters: { - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this); - GPRReg scopeGPR = scope.gpr(); - GPRReg resultGPR = result.gpr(); - - m_jit.loadPtr(JITCompiler::Address(scopeGPR, JSVariableObject::offsetOfRegisters()), resultGPR); - storageResult(resultGPR, node); - break; - } - case GetScopedVar: { - StorageOperand registers(this, node->child1()); + case GetClosureVar: { + SpeculateCellOperand base(this, node->child1()); GPRTemporary result(this); - GPRReg registersGPR = registers.gpr(); + GPRReg baseGPR = base.gpr(); GPRReg resultGPR = result.gpr(); - m_jit.load64(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register)), resultGPR); + m_jit.load64(JITCompiler::Address(baseGPR, JSEnvironmentRecord::offsetOfVariable(node->scopeOffset())), resultGPR); jsValueResult(resultGPR, node); break; } - case PutScopedVar: { - SpeculateCellOperand scope(this, node->child1()); - StorageOperand registers(this, node->child2()); - JSValueOperand value(this, node->child3()); - GPRTemporary scratchRegister(this); + case PutClosureVar: { + SpeculateCellOperand base(this, node->child1()); + JSValueOperand value(this, node->child2()); - GPRReg scopeGPR = scope.gpr(); - GPRReg registersGPR = registers.gpr(); + GPRReg baseGPR = base.gpr(); GPRReg valueGPR = value.gpr(); - GPRReg scratchGPR = scratchRegister.gpr(); - m_jit.store64(valueGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register))); - writeBarrier(scopeGPR, valueGPR, node->child3(), WriteBarrierForVariableAccess, scratchGPR); + m_jit.store64(valueGPR, JITCompiler::Address(baseGPR, JSEnvironmentRecord::offsetOfVariable(node->scopeOffset()))); noResult(node); break; } case GetById: { - if (!node->prediction()) { - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); - break; - } + ASSERT(node->prediction()); switch (node->child1().useKind()) { case CellUse: { SpeculateCellOperand base(this, node->child1()); - GPRTemporary result(this, base); + GPRTemporary result(this, Reuse, base); GPRReg baseGPR = base.gpr(); GPRReg resultGPR = result.gpr(); base.use(); - cachedGetById(node->codeOrigin, baseGPR, resultGPR, node->identifierNumber()); + cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber()); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; @@ -3797,23 +3759,23 @@ void SpeculativeJIT::compile(Node* node) case UntypedUse: { JSValueOperand base(this, node->child1()); - GPRTemporary result(this, base); + GPRTemporary result(this, Reuse, base); GPRReg baseGPR = base.gpr(); GPRReg resultGPR = result.gpr(); base.use(); - JITCompiler::Jump notCell = m_jit.branchTest64(JITCompiler::NonZero, baseGPR, GPRInfo::tagMaskRegister); + JITCompiler::Jump notCell = m_jit.branchIfNotCell(JSValueRegs(baseGPR)); - cachedGetById(node->codeOrigin, baseGPR, resultGPR, node->identifierNumber(), notCell); + cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), notCell); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } break; @@ -3830,7 +3792,7 @@ void SpeculativeJIT::compile(Node* node) SpeculateCellOperand base(this, node->child1()); GPRReg baseGPR = base.gpr(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); @@ -3838,7 +3800,7 @@ void SpeculativeJIT::compile(Node* node) flushRegisters(); - cachedGetById(node->codeOrigin, baseGPR, resultGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill); + cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; @@ -3848,22 +3810,22 @@ void SpeculativeJIT::compile(Node* node) JSValueOperand base(this, node->child1()); GPRReg baseGPR = base.gpr(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); base.use(); flushRegisters(); - JITCompiler::Jump notCell = m_jit.branchTest64(JITCompiler::NonZero, baseGPR, GPRInfo::tagMaskRegister); + JITCompiler::Jump notCell = m_jit.branchIfNotCell(JSValueRegs(baseGPR)); - cachedGetById(node->codeOrigin, baseGPR, resultGPR, node->identifierNumber(), notCell, DontSpill); + cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), notCell, DontSpill); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } break; @@ -3873,53 +3835,60 @@ void SpeculativeJIT::compile(Node* node) compileGetArrayLength(node); break; - case CheckFunction: { - SpeculateCellOperand function(this, node->child1()); - speculationCheck(BadFunction, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, function.gpr(), node->function())); + case CheckCell: { + SpeculateCellOperand cell(this, node->child1()); + speculationCheck(BadCell, JSValueSource::unboxedCell(cell.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, cell.gpr(), node->cellOperand()->cell())); noResult(node); break; } - - case CheckExecutable: { - SpeculateCellOperand function(this, node->child1()); - speculationCheck(BadExecutable, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, JITCompiler::Address(function.gpr(), JSFunction::offsetOfExecutable()), node->executable())); + + case CheckNotEmpty: { + JSValueOperand operand(this, node->child1()); + GPRReg gpr = operand.gpr(); + speculationCheck(TDZFailure, JSValueSource(), nullptr, m_jit.branchTest64(JITCompiler::Zero, gpr)); noResult(node); break; } + + case GetExecutable: { + SpeculateCellOperand function(this, node->child1()); + GPRTemporary result(this, Reuse, function); + GPRReg functionGPR = function.gpr(); + GPRReg resultGPR = result.gpr(); + speculateCellType(node->child1(), functionGPR, SpecFunction, JSFunctionType); + m_jit.loadPtr(JITCompiler::Address(functionGPR, JSFunction::offsetOfExecutable()), resultGPR); + cellResult(resultGPR, node); + break; + } - case CheckStructure: - case ForwardCheckStructure: { + case CheckStructure: { SpeculateCellOperand base(this, node->child1()); ASSERT(node->structureSet().size()); ExitKind exitKind; - if (node->child1()->op() == WeakJSConstant) - exitKind = BadWeakConstantCache; + if (node->child1()->hasConstant()) + exitKind = BadConstantCache; else exitKind = BadCache; if (node->structureSet().size() == 1) { speculationCheck( exitKind, JSValueSource::unboxedCell(base.gpr()), 0, - m_jit.branchWeakPtr( + m_jit.branchWeakStructure( JITCompiler::NotEqual, - JITCompiler::Address(base.gpr(), JSCell::structureOffset()), + JITCompiler::Address(base.gpr(), JSCell::structureIDOffset()), node->structureSet()[0])); } else { - GPRTemporary structure(this); - - m_jit.loadPtr(JITCompiler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr()); - JITCompiler::JumpList done; for (size_t i = 0; i < node->structureSet().size() - 1; ++i) - done.append(m_jit.branchWeakPtr(JITCompiler::Equal, structure.gpr(), node->structureSet()[i])); + done.append(m_jit.branchWeakStructure(JITCompiler::Equal, MacroAssembler::Address(base.gpr(), JSCell::structureIDOffset()), node->structureSet()[i])); speculationCheck( exitKind, JSValueSource::unboxedCell(base.gpr()), 0, - m_jit.branchWeakPtr( - JITCompiler::NotEqual, structure.gpr(), node->structureSet().last())); + m_jit.branchWeakStructure( + JITCompiler::NotEqual, MacroAssembler::Address(base.gpr(), JSCell::structureIDOffset()), node->structureSet().last())); done.link(&m_jit); } @@ -3928,62 +3897,19 @@ void SpeculativeJIT::compile(Node* node) break; } - case StructureTransitionWatchpoint: - case ForwardStructureTransitionWatchpoint: { - // There is a fascinating question here of what to do about array profiling. - // We *could* try to tell the OSR exit about where the base of the access is. - // The DFG will have kept it alive, though it may not be in a register, and - // we shouldn't really load it since that could be a waste. For now though, - // we'll just rely on the fact that when a watchpoint fires then that's - // quite a hint already. - - m_jit.addWeakReference(node->structure()); - node->structure()->addTransitionWatchpoint( - speculationWatchpoint( - node->child1()->op() == WeakJSConstant ? BadWeakConstantCache : BadCache)); - -#if !ASSERT_DISABLED - SpeculateCellOperand op1(this, node->child1()); - JITCompiler::Jump isOK = m_jit.branchPtr(JITCompiler::Equal, JITCompiler::Address(op1.gpr(), JSCell::structureOffset()), TrustedImmPtr(node->structure())); - m_jit.breakpoint(); - isOK.link(&m_jit); -#else - speculateCell(node->child1()); -#endif - - noResult(node); - break; - } - - case PhantomPutStructure: { - ASSERT(isKnownCell(node->child1().node())); - - ASSERT(node->structureTransitionData().previousStructure->transitionWatchpointSetHasBeenInvalidated()); - m_jit.addWeakReferenceTransition( - node->codeOrigin.codeOriginOwner(), - node->structureTransitionData().previousStructure, - node->structureTransitionData().newStructure); - noResult(node); - break; - } - case PutStructure: { - ASSERT(node->structureTransitionData().previousStructure->transitionWatchpointSetHasBeenInvalidated()); + Structure* oldStructure = node->transition()->previous; + Structure* newStructure = node->transition()->next; + + m_jit.jitCode()->common.notifyCompilingStructureTransition(m_jit.graph().m_plan, m_jit.codeBlock(), node); SpeculateCellOperand base(this, node->child1()); GPRReg baseGPR = base.gpr(); - m_jit.addWeakReferenceTransition( - node->codeOrigin.codeOriginOwner(), - node->structureTransitionData().previousStructure, - node->structureTransitionData().newStructure); - -#if ENABLE(WRITE_BARRIER_PROFILING) - // Must always emit this write barrier as the structure transition itself requires it - writeBarrier(baseGPR, node->structureTransitionData().newStructure, WriteBarrierForGenericAccess); -#endif - - m_jit.storePtr(MacroAssembler::TrustedImmPtr(node->structureTransitionData().newStructure), MacroAssembler::Address(baseGPR, JSCell::structureOffset())); + ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType()); + ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type()); + ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags()); + m_jit.store32(MacroAssembler::TrustedImm32(newStructure->id()), MacroAssembler::Address(baseGPR, JSCell::structureIDOffset())); noResult(node); break; @@ -3999,7 +3925,7 @@ void SpeculativeJIT::compile(Node* node) case GetButterfly: { SpeculateCellOperand base(this, node->child1()); - GPRTemporary result(this, base); + GPRTemporary result(this, Reuse, base); GPRReg baseGPR = base.gpr(); GPRReg resultGPR = result.gpr(); @@ -4015,39 +3941,89 @@ void SpeculativeJIT::compile(Node* node) break; } - case GetByOffset: { + case ConstantStoragePointer: { + compileConstantStoragePointer(node); + break; + } + + case GetTypedArrayByteOffset: { + compileGetTypedArrayByteOffset(node); + break; + } + + case GetByOffset: + case GetGetterSetterByOffset: { StorageOperand storage(this, node->child1()); - GPRTemporary result(this, storage); + GPRTemporary result(this, Reuse, storage); GPRReg storageGPR = storage.gpr(); GPRReg resultGPR = result.gpr(); - StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()]; + StorageAccessData& storageAccessData = node->storageAccessData(); - m_jit.load64(JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue)), resultGPR); + m_jit.load64(JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset)), resultGPR); jsValueResult(resultGPR, node); break; } + case GetGetter: { + SpeculateCellOperand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); + + GPRReg op1GPR = op1.gpr(); + GPRReg resultGPR = result.gpr(); + + m_jit.loadPtr(JITCompiler::Address(op1GPR, GetterSetter::offsetOfGetter()), resultGPR); + + cellResult(resultGPR, node); + break; + } + + case GetSetter: { + SpeculateCellOperand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); + + GPRReg op1GPR = op1.gpr(); + GPRReg resultGPR = result.gpr(); + + m_jit.loadPtr(JITCompiler::Address(op1GPR, GetterSetter::offsetOfSetter()), resultGPR); + + cellResult(resultGPR, node); + break; + } + case PutByOffset: { -#if ENABLE(WRITE_BARRIER_PROFILING) - SpeculateCellOperand base(this, node->child2()); -#endif StorageOperand storage(this, node->child1()); JSValueOperand value(this, node->child3()); + GPRTemporary scratch1(this); + GPRTemporary scratch2(this); GPRReg storageGPR = storage.gpr(); GPRReg valueGPR = value.gpr(); - -#if ENABLE(WRITE_BARRIER_PROFILING) - writeBarrier(base.gpr(), value.gpr(), node->child3(), WriteBarrierForPropertyAccess); -#endif - StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()]; - - m_jit.store64(valueGPR, JITCompiler::Address(storageGPR, storageAccessData.offset * sizeof(EncodedJSValue))); + speculate(node, node->child2()); + + StorageAccessData& storageAccessData = node->storageAccessData(); + m_jit.store64(valueGPR, JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset))); + + noResult(node); + break; + } + + case PutByIdFlush: { + SpeculateCellOperand base(this, node->child1()); + JSValueOperand value(this, node->child2()); + GPRTemporary scratch(this); + + GPRReg baseGPR = base.gpr(); + GPRReg valueGPR = value.gpr(); + GPRReg scratchGPR = scratch.gpr(); + flushRegisters(); + + cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), NotDirect, MacroAssembler::Jump(), DontSpill); + noResult(node); break; } @@ -4061,12 +4037,9 @@ void SpeculativeJIT::compile(Node* node) GPRReg valueGPR = value.gpr(); GPRReg scratchGPR = scratch.gpr(); - base.use(); - value.use(); + cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), NotDirect); - cachedPutById(node->codeOrigin, baseGPR, valueGPR, node->child2(), scratchGPR, node->identifierNumber(), NotDirect); - - noResult(node, UseChildrenCalledExplicitly); + noResult(node); break; } @@ -4079,82 +4052,36 @@ void SpeculativeJIT::compile(Node* node) GPRReg valueGPR = value.gpr(); GPRReg scratchGPR = scratch.gpr(); - base.use(); - value.use(); + cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), Direct); - cachedPutById(node->codeOrigin, baseGPR, valueGPR, node->child2(), scratchGPR, node->identifierNumber(), Direct); - - noResult(node, UseChildrenCalledExplicitly); + noResult(node); break; } case GetGlobalVar: { GPRTemporary result(this); - m_jit.load64(node->registerPointer(), result.gpr()); + m_jit.load64(node->variablePointer(), result.gpr()); jsValueResult(result.gpr(), node); break; } case PutGlobalVar: { - JSValueOperand value(this, node->child1()); - - if (Heap::isWriteBarrierEnabled()) { - GPRTemporary scratch(this); - GPRReg scratchReg = scratch.gpr(); - - writeBarrier(m_jit.globalObjectFor(node->codeOrigin), value.gpr(), node->child1(), WriteBarrierForVariableAccess, scratchReg); - } - - m_jit.store64(value.gpr(), node->registerPointer()); + JSValueOperand value(this, node->child2()); + + m_jit.store64(value.gpr(), node->variablePointer()); noResult(node); break; } - case PutGlobalVarCheck: { - JSValueOperand value(this, node->child1()); - - WatchpointSet* watchpointSet = - m_jit.globalObjectFor(node->codeOrigin)->symbolTable()->get( - identifier(node->identifierNumberForCheck())->impl()).watchpointSet(); - addSlowPathGenerator( - slowPathCall( - m_jit.branchTest8( - JITCompiler::NonZero, - JITCompiler::AbsoluteAddress(watchpointSet->addressOfIsWatched())), - this, operationNotifyGlobalVarWrite, NoResult, watchpointSet)); - - if (Heap::isWriteBarrierEnabled()) { - GPRTemporary scratch(this); - GPRReg scratchReg = scratch.gpr(); - - writeBarrier(m_jit.globalObjectFor(node->codeOrigin), value.gpr(), node->child1(), WriteBarrierForVariableAccess, scratchReg); - } - - m_jit.store64(value.gpr(), node->registerPointer()); - - noResult(node); + case NotifyWrite: { + compileNotifyWrite(node); break; } - - case GlobalVarWatchpoint: { - m_jit.globalObjectFor(node->codeOrigin)->symbolTable()->get( - identifier(node->identifierNumberForCheck())->impl()).addWatchpoint( - speculationWatchpoint()); - -#if DFG_ENABLE(JIT_ASSERT) - GPRTemporary scratch(this); - GPRReg scratchGPR = scratch.gpr(); - m_jit.load64(node->registerPointer(), scratchGPR); - JITCompiler::Jump ok = m_jit.branch64( - JITCompiler::Equal, scratchGPR, - TrustedImm64(JSValue::encode(node->registerPointer()->get()))); - m_jit.breakpoint(); - ok.link(&m_jit); -#endif - + + case VarInjectionWatchpoint: { noResult(node); break; } @@ -4164,8 +4091,10 @@ void SpeculativeJIT::compile(Node* node) GPRTemporary structure(this); // Speculate that base 'ImplementsDefaultHasInstance'. - m_jit.loadPtr(MacroAssembler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr()); - speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(structure.gpr(), Structure::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance))); + speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branchTest8( + MacroAssembler::Zero, + MacroAssembler::Address(base.gpr(), JSCell::typeInfoFlagsOffset()), + MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance))); noResult(node); break; @@ -4181,28 +4110,31 @@ void SpeculativeJIT::compile(Node* node) GPRTemporary result(this); GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); + GPRTemporary scratch(this); - JITCompiler::Jump isCell = m_jit.branchTest64(JITCompiler::Zero, value.gpr(), GPRInfo::tagMaskRegister); + JITCompiler::Jump isCell = m_jit.branchIfCell(value.jsValueRegs()); m_jit.compare64(JITCompiler::Equal, value.gpr(), TrustedImm32(ValueUndefined), result.gpr()); JITCompiler::Jump done = m_jit.jump(); isCell.link(&m_jit); JITCompiler::Jump notMasqueradesAsUndefined; - if (m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->isStillValid()) { - m_jit.graph().globalObjectFor(node->codeOrigin)->masqueradesAsUndefinedWatchpoint()->add(speculationWatchpoint()); + if (masqueradesAsUndefinedWatchpointIsStillValid()) { m_jit.move(TrustedImm32(0), result.gpr()); notMasqueradesAsUndefined = m_jit.jump(); } else { - m_jit.loadPtr(JITCompiler::Address(value.gpr(), JSCell::structureOffset()), result.gpr()); - JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(result.gpr(), Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)); + JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::NonZero, + JITCompiler::Address(value.gpr(), JSCell::typeInfoFlagsOffset()), + TrustedImm32(MasqueradesAsUndefined)); m_jit.move(TrustedImm32(0), result.gpr()); notMasqueradesAsUndefined = m_jit.jump(); isMasqueradesAsUndefined.link(&m_jit); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(TrustedImmPtr(m_jit.globalObjectFor(node->codeOrigin)), localGlobalObjectGPR); + m_jit.move(TrustedImmPtr(m_jit.globalObjectFor(node->origin.semantic)), localGlobalObjectGPR); + m_jit.emitLoadStructure(value.gpr(), result.gpr(), scratch.gpr()); m_jit.loadPtr(JITCompiler::Address(result.gpr(), Structure::globalObjectOffset()), remoteGlobalObjectGPR); m_jit.comparePtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, result.gpr()); } @@ -4216,7 +4148,7 @@ void SpeculativeJIT::compile(Node* node) case IsBoolean: { JSValueOperand value(this, node->child1()); - GPRTemporary result(this, value); + GPRTemporary result(this, Reuse, value); m_jit.move(value.gpr(), result.gpr()); m_jit.xor64(JITCompiler::TrustedImm32(ValueFalse), result.gpr()); @@ -4228,7 +4160,7 @@ void SpeculativeJIT::compile(Node* node) case IsNumber: { JSValueOperand value(this, node->child1()); - GPRTemporary result(this, value); + GPRTemporary result(this, Reuse, value); m_jit.test64(JITCompiler::NonZero, value.gpr(), GPRInfo::tagTypeNumberRegister, result.gpr()); m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); @@ -4238,12 +4170,14 @@ void SpeculativeJIT::compile(Node* node) case IsString: { JSValueOperand value(this, node->child1()); - GPRTemporary result(this, value); + GPRTemporary result(this, Reuse, value); - JITCompiler::Jump isNotCell = m_jit.branchTest64(JITCompiler::NonZero, value.gpr(), GPRInfo::tagMaskRegister); + JITCompiler::Jump isNotCell = m_jit.branchIfNotCell(value.jsValueRegs()); - m_jit.loadPtr(JITCompiler::Address(value.gpr(), JSCell::structureOffset()), result.gpr()); - m_jit.compare8(JITCompiler::Equal, JITCompiler::Address(result.gpr(), Structure::typeInfoTypeOffset()), TrustedImm32(StringType), result.gpr()); + m_jit.compare8(JITCompiler::Equal, + JITCompiler::Address(value.gpr(), JSCell::typeInfoTypeOffset()), + TrustedImm32(StringType), + result.gpr()); m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); JITCompiler::Jump done = m_jit.jump(); @@ -4254,564 +4188,592 @@ void SpeculativeJIT::compile(Node* node) jsValueResult(result.gpr(), node, DataFormatJSBoolean); break; } - + case IsObject: { JSValueOperand value(this, node->child1()); - GPRReg valueGPR = value.gpr(); - GPRResult result(this); - GPRReg resultGPR = result.gpr(); - flushRegisters(); - callOperation(operationIsObject, resultGPR, valueGPR); - m_jit.or32(TrustedImm32(ValueFalse), resultGPR); + GPRTemporary result(this, Reuse, value); + + JITCompiler::Jump isNotCell = m_jit.branchIfNotCell(value.jsValueRegs()); + + m_jit.compare8(JITCompiler::AboveOrEqual, + JITCompiler::Address(value.gpr(), JSCell::typeInfoTypeOffset()), + TrustedImm32(ObjectType), + result.gpr()); + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + JITCompiler::Jump done = m_jit.jump(); + + isNotCell.link(&m_jit); + m_jit.move(TrustedImm32(ValueFalse), result.gpr()); + + done.link(&m_jit); jsValueResult(result.gpr(), node, DataFormatJSBoolean); break; } + case IsObjectOrNull: { + compileIsObjectOrNull(node); + break; + } + case IsFunction: { - JSValueOperand value(this, node->child1()); - GPRReg valueGPR = value.gpr(); - GPRResult result(this); - GPRReg resultGPR = result.gpr(); - flushRegisters(); - callOperation(operationIsFunction, resultGPR, valueGPR); - m_jit.or32(TrustedImm32(ValueFalse), resultGPR); - jsValueResult(result.gpr(), node, DataFormatJSBoolean); + compileIsFunction(node); break; } case TypeOf: { - JSValueOperand value(this, node->child1(), ManualOperandSpeculation); - GPRReg valueGPR = value.gpr(); - GPRTemporary temp(this); - GPRReg tempGPR = temp.gpr(); - GPRResult result(this); - GPRReg resultGPR = result.gpr(); - JITCompiler::JumpList doneJumps; - - flushRegisters(); - - ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == CellUse || node->child1().useKind() == StringUse); - - JITCompiler::Jump isNotCell = m_jit.branchTest64(JITCompiler::NonZero, valueGPR, GPRInfo::tagMaskRegister); - if (node->child1().useKind() != UntypedUse) - DFG_TYPE_CHECK(JSValueSource(valueGPR), node->child1(), SpecCell, isNotCell); - - if (!node->child1()->shouldSpeculateObject() || node->child1().useKind() == StringUse) { - m_jit.loadPtr(JITCompiler::Address(valueGPR, JSCell::structureOffset()), tempGPR); - JITCompiler::Jump notString = m_jit.branch8(JITCompiler::NotEqual, JITCompiler::Address(tempGPR, Structure::typeInfoTypeOffset()), TrustedImm32(StringType)); - if (node->child1().useKind() == StringUse) - DFG_TYPE_CHECK(JSValueSource(valueGPR), node->child1(), SpecString, notString); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.stringString()), resultGPR); - doneJumps.append(m_jit.jump()); - if (node->child1().useKind() != StringUse) { - notString.link(&m_jit); - callOperation(operationTypeOf, resultGPR, valueGPR); - doneJumps.append(m_jit.jump()); - } - } else { - callOperation(operationTypeOf, resultGPR, valueGPR); - doneJumps.append(m_jit.jump()); - } - - if (node->child1().useKind() == UntypedUse) { - isNotCell.link(&m_jit); - JITCompiler::Jump notNumber = m_jit.branchTest64(JITCompiler::Zero, valueGPR, GPRInfo::tagTypeNumberRegister); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.numberString()), resultGPR); - doneJumps.append(m_jit.jump()); - notNumber.link(&m_jit); - - JITCompiler::Jump notUndefined = m_jit.branch64(JITCompiler::NotEqual, valueGPR, JITCompiler::TrustedImm64(ValueUndefined)); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.undefinedString()), resultGPR); - doneJumps.append(m_jit.jump()); - notUndefined.link(&m_jit); - - JITCompiler::Jump notNull = m_jit.branch64(JITCompiler::NotEqual, valueGPR, JITCompiler::TrustedImm64(ValueNull)); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.objectString()), resultGPR); - doneJumps.append(m_jit.jump()); - notNull.link(&m_jit); - - // Only boolean left - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.booleanString()), resultGPR); - } - doneJumps.link(&m_jit); - cellResult(resultGPR, node); + compileTypeOf(node); break; } case Flush: - case Phi: break; - case Breakpoint: -#if ENABLE(DEBUG_WITH_BREAKPOINT) - m_jit.breakpoint(); -#else - RELEASE_ASSERT_NOT_REACHED(); -#endif - break; - case Call: case Construct: + case CallVarargs: + case CallForwardVarargs: + case ConstructVarargs: + case ConstructForwardVarargs: emitCall(node); break; + + case LoadVarargs: { + LoadVarargsData* data = node->loadVarargsData(); + + GPRReg argumentsGPR; + { + JSValueOperand arguments(this, node->child1()); + argumentsGPR = arguments.gpr(); + flushRegisters(); + } + + callOperation(operationSizeOfVarargs, GPRInfo::returnValueGPR, argumentsGPR, data->offset); + + lock(GPRInfo::returnValueGPR); + { + JSValueOperand arguments(this, node->child1()); + argumentsGPR = arguments.gpr(); + flushRegisters(); + } + unlock(GPRInfo::returnValueGPR); + + // FIXME: There is a chance that we will call an effectful length property twice. This is safe + // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance + // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right + // past the sizing. + // https://bugs.webkit.org/show_bug.cgi?id=141448 - case Resolve: { - flushRegisters(); - GPRResult result(this); - ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()]; - callOperation(operationResolve, result.gpr(), identifier(data.identifierNumber), data.resolveOperations); - jsValueResult(result.gpr(), node); + GPRReg argCountIncludingThisGPR = + JITCompiler::selectScratchGPR(GPRInfo::returnValueGPR, argumentsGPR); + + m_jit.add32(TrustedImm32(1), GPRInfo::returnValueGPR, argCountIncludingThisGPR); + speculationCheck( + VarargsOverflow, JSValueSource(), Edge(), m_jit.branch32( + MacroAssembler::Above, + argCountIncludingThisGPR, + TrustedImm32(data->limit))); + + m_jit.store32(argCountIncludingThisGPR, JITCompiler::payloadFor(data->machineCount)); + + callOperation(operationLoadVarargs, data->machineStart.offset(), argumentsGPR, data->offset, GPRInfo::returnValueGPR, data->mandatoryMinimum); + + noResult(node); break; } - - case ResolveBase: { - flushRegisters(); - GPRResult result(this); - ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()]; - callOperation(operationResolveBase, result.gpr(), identifier(data.identifierNumber), data.resolveOperations, data.putToBaseOperation); - jsValueResult(result.gpr(), node); + + case ForwardVarargs: { + compileForwardVarargs(node); break; } - - case ResolveBaseStrictPut: { - flushRegisters(); - GPRResult result(this); - ResolveOperationData& data = m_jit.graph().m_resolveOperationsData[node->resolveOperationsDataIndex()]; - callOperation(operationResolveBaseStrictPut, result.gpr(), identifier(data.identifierNumber), data.resolveOperations, data.putToBaseOperation); - jsValueResult(result.gpr(), node); + + case CreateActivation: { + compileCreateActivation(node); break; } - - case ResolveGlobal: { - GPRTemporary globalObject(this); - GPRTemporary resolveInfo(this); - GPRTemporary result(this); - - GPRReg globalObjectGPR = globalObject.gpr(); - GPRReg resolveInfoGPR = resolveInfo.gpr(); - GPRReg resultGPR = result.gpr(); - - ResolveGlobalData& data = m_jit.graph().m_resolveGlobalData[node->resolveGlobalDataIndex()]; - ResolveOperation* resolveOperationAddress = &(data.resolveOperations->data()[data.resolvePropertyIndex]); - - // Check Structure of global object - m_jit.move(JITCompiler::TrustedImmPtr(m_jit.globalObjectFor(node->codeOrigin)), globalObjectGPR); - m_jit.move(JITCompiler::TrustedImmPtr(resolveOperationAddress), resolveInfoGPR); - m_jit.loadPtr(JITCompiler::Address(resolveInfoGPR, OBJECT_OFFSETOF(ResolveOperation, m_structure)), resultGPR); - JITCompiler::Jump structuresDontMatch = m_jit.branchPtr(JITCompiler::NotEqual, resultGPR, JITCompiler::Address(globalObjectGPR, JSCell::structureOffset())); - - // Fast case - m_jit.load32(JITCompiler::Address(resolveInfoGPR, OBJECT_OFFSETOF(ResolveOperation, m_offset)), resolveInfoGPR); -#if DFG_ENABLE(JIT_ASSERT) - JITCompiler::Jump isOutOfLine = m_jit.branch32(JITCompiler::GreaterThanOrEqual, resolveInfoGPR, TrustedImm32(firstOutOfLineOffset)); - m_jit.breakpoint(); - isOutOfLine.link(&m_jit); -#endif - m_jit.neg32(resolveInfoGPR); - m_jit.signExtend32ToPtr(resolveInfoGPR, resolveInfoGPR); - m_jit.loadPtr(JITCompiler::Address(globalObjectGPR, JSObject::butterflyOffset()), resultGPR); - m_jit.load64(JITCompiler::BaseIndex(resultGPR, resolveInfoGPR, JITCompiler::TimesEight, (firstOutOfLineOffset - 2) * static_cast(sizeof(JSValue))), resultGPR); - addSlowPathGenerator( - slowPathCall( - structuresDontMatch, this, operationResolveGlobal, - resultGPR, resolveInfoGPR, globalObjectGPR, - &m_jit.codeBlock()->identifier(data.identifierNumber))); - - jsValueResult(resultGPR, node); + case CreateDirectArguments: { + compileCreateDirectArguments(node); break; } - case CreateActivation: { - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); + case GetFromArguments: { + compileGetFromArguments(node); + break; + } - JSValueOperand value(this, node->child1()); - GPRTemporary result(this, value); + case PutToArguments: { + compilePutToArguments(node); + break; + } - GPRReg valueGPR = value.gpr(); - GPRReg resultGPR = result.gpr(); + case CreateScopedArguments: { + compileCreateScopedArguments(node); + break; + } - m_jit.move(valueGPR, resultGPR); + case CreateClonedArguments: { + compileCreateClonedArguments(node); + break; + } - JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR); + case NewFunction: + compileNewFunction(node); + break; - addSlowPathGenerator( - slowPathCall(notCreated, this, operationCreateActivation, resultGPR)); + case In: + compileIn(node); + break; - cellResult(resultGPR, node); + case CountExecution: + m_jit.add64(TrustedImm32(1), MacroAssembler::AbsoluteAddress(node->executionCounter()->address())); + break; + + case ForceOSRExit: { + terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); break; } - case CreateArguments: { - JSValueOperand value(this, node->child1()); - GPRTemporary result(this, value); + case InvalidationPoint: + emitInvalidationPoint(node); + break; + + case CheckWatchdogTimer: + ASSERT(m_jit.vm()->watchdog); + speculationCheck( + WatchdogTimerFired, JSValueRegs(), 0, + m_jit.branchTest8( + JITCompiler::NonZero, + JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog->timerDidFireAddress()))); + break; + + case Phantom: + case Check: + DFG_NODE_DO_TO_CHILDREN(m_jit.graph(), node, speculate); + noResult(node); + break; - GPRReg valueGPR = value.gpr(); + case Breakpoint: + case ProfileWillCall: + case ProfileDidCall: + case PhantomLocal: + case LoopHint: + // This is a no-op. + noResult(node); + break; + + case Unreachable: + DFG_CRASH(m_jit.graph(), node, "Unexpected Unreachable node"); + break; + + case StoreBarrier: { + compileStoreBarrier(node); + break; + } + + case GetEnumerableLength: { + SpeculateCellOperand enumerator(this, node->child1()); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); - - m_jit.move(valueGPR, resultGPR); - - JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR); - - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - notCreated, this, operationCreateInlinedArguments, resultGPR, - node->codeOrigin.inlineCallFrame)); - } else { - addSlowPathGenerator( - slowPathCall(notCreated, this, operationCreateArguments, resultGPR)); - } - - cellResult(resultGPR, node); + + m_jit.load32(MacroAssembler::Address(enumerator.gpr(), JSPropertyNameEnumerator::indexedLengthOffset()), resultGPR); + int32Result(resultGPR, node); break; } + case HasGenericProperty: { + JSValueOperand base(this, node->child1()); + SpeculateCellOperand property(this, node->child2()); + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); - case TearOffActivation: { - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); + flushRegisters(); + callOperation(operationHasGenericProperty, resultGPR, base.gpr(), property.gpr()); + jsValueResult(resultGPR, node, DataFormatJSBoolean); + break; + } + case HasStructureProperty: { + JSValueOperand base(this, node->child1()); + SpeculateCellOperand property(this, node->child2()); + SpeculateCellOperand enumerator(this, node->child3()); + GPRTemporary result(this); - JSValueOperand activationValue(this, node->child1()); - GPRTemporary scratch(this); - GPRReg activationValueGPR = activationValue.gpr(); - GPRReg scratchGPR = scratch.gpr(); + GPRReg baseGPR = base.gpr(); + GPRReg propertyGPR = property.gpr(); + GPRReg resultGPR = result.gpr(); - JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, activationValueGPR); + m_jit.load32(MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()), resultGPR); + MacroAssembler::Jump wrongStructure = m_jit.branch32(MacroAssembler::NotEqual, + resultGPR, + MacroAssembler::Address(enumerator.gpr(), JSPropertyNameEnumerator::cachedStructureIDOffset())); - SharedSymbolTable* symbolTable = m_jit.symbolTableFor(node->codeOrigin); - int registersOffset = JSActivation::registersOffset(symbolTable); + moveTrueTo(resultGPR); + MacroAssembler::Jump done = m_jit.jump(); - int captureEnd = symbolTable->captureEnd(); - for (int i = symbolTable->captureStart(); i < captureEnd; ++i) { - m_jit.load64( - JITCompiler::Address( - GPRInfo::callFrameRegister, i * sizeof(Register)), scratchGPR); - m_jit.store64( - scratchGPR, JITCompiler::Address( - activationValueGPR, registersOffset + i * sizeof(Register))); - } - m_jit.addPtr(TrustedImm32(registersOffset), activationValueGPR, scratchGPR); - m_jit.storePtr(scratchGPR, JITCompiler::Address(activationValueGPR, JSActivation::offsetOfRegisters())); + done.link(&m_jit); - notCreated.link(&m_jit); - noResult(node); + addSlowPathGenerator(slowPathCall(wrongStructure, this, operationHasGenericProperty, resultGPR, baseGPR, propertyGPR)); + jsValueResult(resultGPR, node, DataFormatJSBoolean); break; } + case HasIndexedProperty: { + SpeculateCellOperand base(this, node->child1()); + SpeculateStrictInt32Operand index(this, node->child2()); + GPRTemporary result(this); - case TearOffArguments: { - JSValueOperand unmodifiedArgumentsValue(this, node->child1()); - JSValueOperand activationValue(this, node->child2()); - GPRReg unmodifiedArgumentsValueGPR = unmodifiedArgumentsValue.gpr(); - GPRReg activationValueGPR = activationValue.gpr(); + GPRReg baseGPR = base.gpr(); + GPRReg indexGPR = index.gpr(); + GPRReg resultGPR = result.gpr(); - JITCompiler::Jump created = m_jit.branchTest64(JITCompiler::NonZero, unmodifiedArgumentsValueGPR); + MacroAssembler::JumpList slowCases; + ArrayMode mode = node->arrayMode(); + switch (mode.type()) { + case Array::Int32: + case Array::Contiguous: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + GPRTemporary scratch(this); + + GPRReg storageGPR = storage.gpr(); + GPRReg scratchGPR = scratch.gpr(); - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - created, this, operationTearOffInlinedArguments, NoResult, - unmodifiedArgumentsValueGPR, activationValueGPR, node->codeOrigin.inlineCallFrame)); - } else { - addSlowPathGenerator( - slowPathCall( - created, this, operationTearOffArguments, NoResult, unmodifiedArgumentsValueGPR, activationValueGPR)); + MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength())); + if (mode.isInBounds()) + speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds); + else + slowCases.append(outOfBounds); + + m_jit.load64(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), scratchGPR); + slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, scratchGPR)); + moveTrueTo(resultGPR); + break; } + case Array::Double: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + FPRTemporary scratch(this); + FPRReg scratchFPR = scratch.fpr(); + GPRReg storageGPR = storage.gpr(); + + MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength())); + if (mode.isInBounds()) + speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds); + else + slowCases.append(outOfBounds); + + m_jit.loadDouble(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), scratchFPR); + slowCases.append(m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, scratchFPR, scratchFPR)); + moveTrueTo(resultGPR); + break; + } + case Array::ArrayStorage: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + GPRTemporary scratch(this); + + GPRReg storageGPR = storage.gpr(); + GPRReg scratchGPR = scratch.gpr(); + + MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset())); + if (mode.isInBounds()) + speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds); + else + slowCases.append(outOfBounds); + + m_jit.load64(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, ArrayStorage::vectorOffset()), scratchGPR); + slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, scratchGPR)); + moveTrueTo(resultGPR); + break; + } + default: { + slowCases.append(m_jit.jump()); + break; + } + } + + addSlowPathGenerator(slowPathCall(slowCases, this, operationHasIndexedProperty, resultGPR, baseGPR, indexGPR)); - noResult(node); + jsValueResult(resultGPR, node, DataFormatJSBoolean); break; } - - case GetMyArgumentsLength: { + case GetDirectPname: { + Edge& baseEdge = m_jit.graph().varArgChild(node, 0); + Edge& propertyEdge = m_jit.graph().varArgChild(node, 1); + Edge& indexEdge = m_jit.graph().varArgChild(node, 2); + Edge& enumeratorEdge = m_jit.graph().varArgChild(node, 3); + + SpeculateCellOperand base(this, baseEdge); + SpeculateCellOperand property(this, propertyEdge); + SpeculateStrictInt32Operand index(this, indexEdge); + SpeculateCellOperand enumerator(this, enumeratorEdge); GPRTemporary result(this); + GPRTemporary scratch1(this); + GPRTemporary scratch2(this); + + GPRReg baseGPR = base.gpr(); + GPRReg propertyGPR = property.gpr(); + GPRReg indexGPR = index.gpr(); + GPRReg enumeratorGPR = enumerator.gpr(); GPRReg resultGPR = result.gpr(); + GPRReg scratch1GPR = scratch1.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); + + // Check the structure + m_jit.load32(MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()), scratch1GPR); + MacroAssembler::Jump wrongStructure = m_jit.branch32(MacroAssembler::NotEqual, + scratch1GPR, MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedStructureIDOffset())); - if (!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) { - speculationCheck( - ArgumentsEscaped, JSValueRegs(), 0, - m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.argumentsRegisterFor(node->codeOrigin)))); - } + // Compute the offset + // If index is less than the enumerator's cached inline storage, then it's an inline access + MacroAssembler::Jump outOfLineAccess = m_jit.branch32(MacroAssembler::AboveOrEqual, + indexGPR, MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedInlineCapacityOffset())); + + m_jit.load64(MacroAssembler::BaseIndex(baseGPR, indexGPR, MacroAssembler::TimesEight, JSObject::offsetOfInlineStorage()), resultGPR); + + MacroAssembler::Jump done = m_jit.jump(); - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); - m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR); - m_jit.sub32(TrustedImm32(1), resultGPR); - integerResult(resultGPR, node); + // Otherwise it's out of line + outOfLineAccess.link(&m_jit); + m_jit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratch2GPR); + m_jit.move(indexGPR, scratch1GPR); + m_jit.sub32(MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedInlineCapacityOffset()), scratch1GPR); + m_jit.neg32(scratch1GPR); + m_jit.signExtend32ToPtr(scratch1GPR, scratch1GPR); + int32_t offsetOfFirstProperty = static_cast(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue); + m_jit.load64(MacroAssembler::BaseIndex(scratch2GPR, scratch1GPR, MacroAssembler::TimesEight, offsetOfFirstProperty), resultGPR); + + done.link(&m_jit); + + addSlowPathGenerator(slowPathCall(wrongStructure, this, operationGetByVal, resultGPR, baseGPR, propertyGPR)); + + jsValueResult(resultGPR, node); break; } - - case GetMyArgumentsLengthSafe: { - GPRTemporary result(this); + case GetPropertyEnumerator: { + SpeculateCellOperand base(this, node->child1()); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); - - JITCompiler::Jump created = m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.argumentsRegisterFor(node->codeOrigin))); - - if (node->codeOrigin.inlineCallFrame) { - m_jit.move( - Imm64(JSValue::encode(jsNumber(node->codeOrigin.inlineCallFrame->arguments.size() - 1))), - resultGPR); - } else { - m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR); - m_jit.sub32(TrustedImm32(1), resultGPR); - m_jit.or64(GPRInfo::tagTypeNumberRegister, resultGPR); - } - - // FIXME: the slow path generator should perform a forward speculation that the - // result is an integer. For now we postpone the speculation by having this return - // a JSValue. - - addSlowPathGenerator( - slowPathCall( - created, this, operationGetArgumentsLength, resultGPR, - m_jit.argumentsRegisterFor(node->codeOrigin))); - - jsValueResult(resultGPR, node); + + flushRegisters(); + callOperation(operationGetPropertyEnumerator, resultGPR, base.gpr()); + cellResult(resultGPR, node); break; } - - case GetMyArgumentByVal: { - SpeculateStrictInt32Operand index(this, node->child1()); + case GetEnumeratorStructurePname: + case GetEnumeratorGenericPname: { + SpeculateCellOperand enumerator(this, node->child1()); + SpeculateStrictInt32Operand index(this, node->child2()); + GPRTemporary scratch1(this); GPRTemporary result(this); + + GPRReg enumeratorGPR = enumerator.gpr(); GPRReg indexGPR = index.gpr(); + GPRReg scratch1GPR = scratch1.gpr(); GPRReg resultGPR = result.gpr(); - if (!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) { - speculationCheck( - ArgumentsEscaped, JSValueRegs(), 0, - m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.argumentsRegisterFor(node->codeOrigin)))); - } - - m_jit.add32(TrustedImm32(1), indexGPR, resultGPR); - if (node->codeOrigin.inlineCallFrame) { - speculationCheck( - Uncountable, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultGPR, - Imm32(node->codeOrigin.inlineCallFrame->arguments.size()))); - } else { - speculationCheck( - Uncountable, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultGPR, - JITCompiler::payloadFor(JSStack::ArgumentCount))); - } + MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, indexGPR, + MacroAssembler::Address(enumeratorGPR, (op == GetEnumeratorStructurePname) + ? JSPropertyNameEnumerator::endStructurePropertyIndexOffset() + : JSPropertyNameEnumerator::endGenericPropertyIndexOffset())); - JITCompiler::JumpList slowArgument; - JITCompiler::JumpList slowArgumentOutOfBounds; - if (const SlowArgument* slowArguments = m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) { - slowArgumentOutOfBounds.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, indexGPR, - Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount()))); + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsNull())), resultGPR); - COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes); - m_jit.move(ImmPtr(slowArguments), resultGPR); - m_jit.load32( - JITCompiler::BaseIndex( - resultGPR, indexGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(SlowArgument, index)), - resultGPR); - m_jit.signExtend32ToPtr(resultGPR, resultGPR); - m_jit.load64( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight, m_jit.offsetOfLocals(node->codeOrigin)), - resultGPR); - slowArgument.append(m_jit.jump()); - } - slowArgumentOutOfBounds.link(&m_jit); + MacroAssembler::Jump done = m_jit.jump(); + inBounds.link(&m_jit); - m_jit.neg32(resultGPR); - m_jit.signExtend32ToPtr(resultGPR, resultGPR); - - m_jit.load64( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight, m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin)), - resultGPR); + m_jit.loadPtr(MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedPropertyNamesVectorOffset()), scratch1GPR); + m_jit.load64(MacroAssembler::BaseIndex(scratch1GPR, indexGPR, MacroAssembler::TimesEight), resultGPR); - slowArgument.link(&m_jit); + done.link(&m_jit); jsValueResult(resultGPR, node); break; } - - case GetMyArgumentByValSafe: { - SpeculateStrictInt32Operand index(this, node->child1()); - GPRTemporary result(this); - GPRReg indexGPR = index.gpr(); + case ToIndexString: { + SpeculateInt32Operand index(this, node->child1()); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); - - JITCompiler::JumpList slowPath; - slowPath.append( - m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.argumentsRegisterFor(node->codeOrigin)))); - - m_jit.add32(TrustedImm32(1), indexGPR, resultGPR); - if (node->codeOrigin.inlineCallFrame) { - slowPath.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultGPR, - Imm32(node->codeOrigin.inlineCallFrame->arguments.size()))); - } else { - slowPath.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultGPR, - JITCompiler::payloadFor(JSStack::ArgumentCount))); - } - - JITCompiler::JumpList slowArgument; - JITCompiler::JumpList slowArgumentOutOfBounds; - if (const SlowArgument* slowArguments = m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) { - slowArgumentOutOfBounds.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, indexGPR, - Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount()))); - COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes); - m_jit.move(ImmPtr(slowArguments), resultGPR); - m_jit.load32( - JITCompiler::BaseIndex( - resultGPR, indexGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(SlowArgument, index)), - resultGPR); - m_jit.signExtend32ToPtr(resultGPR, resultGPR); - m_jit.load64( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight, m_jit.offsetOfLocals(node->codeOrigin)), - resultGPR); - slowArgument.append(m_jit.jump()); - } - slowArgumentOutOfBounds.link(&m_jit); + flushRegisters(); + callOperation(operationToIndexString, resultGPR, index.gpr()); + cellResult(resultGPR, node); + break; + } + case ProfileType: { + JSValueOperand value(this, node->child1()); + GPRTemporary scratch1(this); + GPRTemporary scratch2(this); + GPRTemporary scratch3(this); - m_jit.neg32(resultGPR); - m_jit.signExtend32ToPtr(resultGPR, resultGPR); - - m_jit.load64( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight, m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin)), - resultGPR); - - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - slowPath, this, operationGetInlinedArgumentByVal, resultGPR, - m_jit.argumentsRegisterFor(node->codeOrigin), - node->codeOrigin.inlineCallFrame, - indexGPR)); - } else { - addSlowPathGenerator( - slowPathCall( - slowPath, this, operationGetArgumentByVal, resultGPR, - m_jit.argumentsRegisterFor(node->codeOrigin), - indexGPR)); + GPRReg scratch1GPR = scratch1.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); + GPRReg scratch3GPR = scratch3.gpr(); + GPRReg valueGPR = value.gpr(); + + MacroAssembler::JumpList jumpToEnd; + + TypeLocation* cachedTypeLocation = node->typeLocation(); + // Compile in a predictive type check, if possible, to see if we can skip writing to the log. + // These typechecks are inlined to match those of the 64-bit JSValue type checks. + if (cachedTypeLocation->m_lastSeenType == TypeUndefined) + jumpToEnd.append(m_jit.branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())))); + else if (cachedTypeLocation->m_lastSeenType == TypeNull) + jumpToEnd.append(m_jit.branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsNull())))); + else if (cachedTypeLocation->m_lastSeenType == TypeBoolean) { + m_jit.move(valueGPR, scratch2GPR); + m_jit.and64(TrustedImm32(~1), scratch2GPR); + jumpToEnd.append(m_jit.branch64(MacroAssembler::Equal, scratch2GPR, MacroAssembler::TrustedImm64(ValueFalse))); + } else if (cachedTypeLocation->m_lastSeenType == TypeMachineInt) + jumpToEnd.append(m_jit.branch64(MacroAssembler::AboveOrEqual, valueGPR, GPRInfo::tagTypeNumberRegister)); + else if (cachedTypeLocation->m_lastSeenType == TypeNumber) + jumpToEnd.append(m_jit.branchTest64(MacroAssembler::NonZero, valueGPR, GPRInfo::tagTypeNumberRegister)); + else if (cachedTypeLocation->m_lastSeenType == TypeString) { + MacroAssembler::Jump isNotCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); + jumpToEnd.append(m_jit.branchIfString(valueGPR)); + isNotCell.link(&m_jit); } - - slowArgument.link(&m_jit); - jsValueResult(resultGPR, node); + + // Load the TypeProfilerLog into Scratch2. + TypeProfilerLog* cachedTypeProfilerLog = m_jit.vm()->typeProfilerLog(); + m_jit.move(TrustedImmPtr(cachedTypeProfilerLog), scratch2GPR); + + // Load the next LogEntry into Scratch1. + m_jit.loadPtr(MacroAssembler::Address(scratch2GPR, TypeProfilerLog::currentLogEntryOffset()), scratch1GPR); + + // Store the JSValue onto the log entry. + m_jit.store64(valueGPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::valueOffset())); + + // Store the structureID of the cell if valueGPR is a cell, otherwise, store 0 on the log entry. + MacroAssembler::Jump isNotCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); + m_jit.load32(MacroAssembler::Address(valueGPR, JSCell::structureIDOffset()), scratch3GPR); + m_jit.store32(scratch3GPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::structureIDOffset())); + MacroAssembler::Jump skipIsCell = m_jit.jump(); + isNotCell.link(&m_jit); + m_jit.store32(TrustedImm32(0), MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::structureIDOffset())); + skipIsCell.link(&m_jit); + + // Store the typeLocation on the log entry. + m_jit.move(TrustedImmPtr(cachedTypeLocation), scratch3GPR); + m_jit.storePtr(scratch3GPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::locationOffset())); + + // Increment the current log entry. + m_jit.addPtr(TrustedImm32(sizeof(TypeProfilerLog::LogEntry)), scratch1GPR); + m_jit.storePtr(scratch1GPR, MacroAssembler::Address(scratch2GPR, TypeProfilerLog::currentLogEntryOffset())); + MacroAssembler::Jump clearLog = m_jit.branchPtr(MacroAssembler::Equal, scratch1GPR, TrustedImmPtr(cachedTypeProfilerLog->logEndPtr())); + addSlowPathGenerator( + slowPathCall(clearLog, this, operationProcessTypeProfilerLogDFG, NoResult)); + + jumpToEnd.link(&m_jit); + + noResult(node); break; } - - case CheckArgumentsNotCreated: { - ASSERT(!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)); - speculationCheck( - ArgumentsEscaped, JSValueRegs(), 0, - m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.argumentsRegisterFor(node->codeOrigin)))); + case ProfileControlFlow: { + BasicBlockLocation* basicBlockLocation = node->basicBlockLocation(); + if (!basicBlockLocation->hasExecuted()) { + GPRTemporary scratch1(this); + basicBlockLocation->emitExecuteCode(m_jit, scratch1.gpr()); + } noResult(node); break; } + +#if ENABLE(FTL_JIT) + case CheckTierUpInLoop: { + MacroAssembler::Jump done = m_jit.branchAdd32( + MacroAssembler::Signed, + TrustedImm32(Options::ftlTierUpCounterIncrementForLoop()), + MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->tierUpCounter.m_counter)); - case NewFunctionNoCheck: - compileNewFunctionNoCheck(node); - break; - - case NewFunction: { - JSValueOperand value(this, node->child1()); - GPRTemporary result(this, value); - - GPRReg valueGPR = value.gpr(); - GPRReg resultGPR = result.gpr(); + silentSpillAllRegisters(InvalidGPRReg); + m_jit.setupArgumentsExecState(); + appendCall(triggerTierUpNowInLoop); + silentFillAllRegisters(InvalidGPRReg); - m_jit.move(valueGPR, resultGPR); + done.link(&m_jit); + break; + } - JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR); + case CheckTierUpAtReturn: { + MacroAssembler::Jump done = m_jit.branchAdd32( + MacroAssembler::Signed, + TrustedImm32(Options::ftlTierUpCounterIncrementForReturn()), + MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->tierUpCounter.m_counter)); - addSlowPathGenerator( - slowPathCall( - notCreated, this, operationNewFunction, - resultGPR, m_jit.codeBlock()->functionDecl(node->functionDeclIndex()))); + silentSpillAllRegisters(InvalidGPRReg); + m_jit.setupArgumentsExecState(); + appendCall(triggerTierUpNow); + silentFillAllRegisters(InvalidGPRReg); - jsValueResult(resultGPR, node); + done.link(&m_jit); break; } - case NewFunctionExpression: - compileNewFunctionExpression(node); - break; + case CheckTierUpAndOSREnter: + case CheckTierUpWithNestedTriggerAndOSREnter: { + ASSERT(!node->origin.semantic.inlineCallFrame); - case CountExecution: - m_jit.add64(TrustedImm32(1), MacroAssembler::AbsoluteAddress(node->executionCounter()->address())); - break; - - case GarbageValue: - // We should never get to the point of code emission for a GarbageValue - CRASH(); - break; + GPRTemporary temp(this); + GPRReg tempGPR = temp.gpr(); - case ForceOSRExit: { - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); + MacroAssembler::Jump forceOSREntry; + if (op == CheckTierUpWithNestedTriggerAndOSREnter) + forceOSREntry = m_jit.branchTest8(MacroAssembler::NonZero, MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->nestedTriggerIsSet)); + + MacroAssembler::Jump done = m_jit.branchAdd32( + MacroAssembler::Signed, + TrustedImm32(Options::ftlTierUpCounterIncrementForLoop()), + MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->tierUpCounter.m_counter)); + + if (forceOSREntry.isSet()) + forceOSREntry.link(&m_jit); + silentSpillAllRegisters(tempGPR); + m_jit.setupArgumentsWithExecState( + TrustedImm32(node->origin.semantic.bytecodeIndex), + TrustedImm32(m_stream->size())); + appendCallSetResult(triggerOSREntryNow, tempGPR); + MacroAssembler::Jump dontEnter = m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR); + m_jit.jump(tempGPR); + dontEnter.link(&m_jit); + silentFillAllRegisters(tempGPR); + + done.link(&m_jit); break; } - - case CheckWatchdogTimer: - speculationCheck( - WatchdogTimerFired, JSValueRegs(), 0, - m_jit.branchTest8( - JITCompiler::NonZero, - JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog.timerDidFireAddress()))); +#else // ENABLE(FTL_JIT) + case CheckTierUpInLoop: + case CheckTierUpAtReturn: + case CheckTierUpAndOSREnter: + case CheckTierUpWithNestedTriggerAndOSREnter: + DFG_CRASH(m_jit.graph(), node, "Unexpected tier-up node"); break; +#endif // ENABLE(FTL_JIT) - case Phantom: - DFG_NODE_DO_TO_CHILDREN(m_jit.graph(), node, speculate); - noResult(node); - break; - - case PhantomLocal: - // This is a no-op. - noResult(node); - break; - - case Nop: - RELEASE_ASSERT_NOT_REACHED(); - break; - + case NativeCall: + case NativeConstruct: case LastNodeType: - RELEASE_ASSERT_NOT_REACHED(); + case Phi: + case Upsilon: + case ExtractOSREntryLocal: + case CheckInBounds: + case ArithIMul: + case MultiGetByOffset: + case MultiPutByOffset: + case FiatInt52: + case CheckBadCell: + case BottomValue: + case PhantomNewObject: + case PhantomNewFunction: + case PhantomCreateActivation: + case GetMyArgumentByVal: + case PutHint: + case CheckStructureImmediate: + case MaterializeNewObject: + case MaterializeCreateActivation: + case PutStack: + case KillStack: + case GetStack: + DFG_CRASH(m_jit.graph(), node, "Unexpected node"); break; } -#if ENABLE(DFG_REGISTER_ALLOCATION_VALIDATION) - m_jit.clearRegisterAllocationOffsets(); -#endif - if (!m_compileOkay) return; @@ -4819,6 +4781,92 @@ void SpeculativeJIT::compile(Node* node) use(node); } +#if ENABLE(GGC) +void SpeculativeJIT::writeBarrier(GPRReg ownerGPR, GPRReg valueGPR, Edge valueUse, GPRReg scratch1, GPRReg scratch2) +{ + JITCompiler::Jump isNotCell; + if (!isKnownCell(valueUse.node())) + isNotCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); + + JITCompiler::Jump ownerIsRememberedOrInEden = m_jit.jumpIfIsRememberedOrInEden(ownerGPR); + storeToWriteBarrierBuffer(ownerGPR, scratch1, scratch2); + ownerIsRememberedOrInEden.link(&m_jit); + + if (!isKnownCell(valueUse.node())) + isNotCell.link(&m_jit); +} +#endif // ENABLE(GGC) + +void SpeculativeJIT::moveTrueTo(GPRReg gpr) +{ + m_jit.move(TrustedImm32(ValueTrue), gpr); +} + +void SpeculativeJIT::moveFalseTo(GPRReg gpr) +{ + m_jit.move(TrustedImm32(ValueFalse), gpr); +} + +void SpeculativeJIT::blessBoolean(GPRReg gpr) +{ + m_jit.or32(TrustedImm32(ValueFalse), gpr); +} + +void SpeculativeJIT::convertMachineInt(Edge valueEdge, GPRReg resultGPR) +{ + JSValueOperand value(this, valueEdge, ManualOperandSpeculation); + GPRReg valueGPR = value.gpr(); + + JITCompiler::Jump notInt32 = + m_jit.branch64(JITCompiler::Below, valueGPR, GPRInfo::tagTypeNumberRegister); + + m_jit.signExtend32ToPtr(valueGPR, resultGPR); + JITCompiler::Jump done = m_jit.jump(); + + notInt32.link(&m_jit); + silentSpillAllRegisters(resultGPR); + callOperation(operationConvertBoxedDoubleToInt52, resultGPR, valueGPR); + silentFillAllRegisters(resultGPR); + + DFG_TYPE_CHECK( + JSValueRegs(valueGPR), valueEdge, SpecInt32 | SpecInt52AsDouble, + m_jit.branch64( + JITCompiler::Equal, resultGPR, + JITCompiler::TrustedImm64(JSValue::notInt52))); + done.link(&m_jit); +} + +void SpeculativeJIT::speculateMachineInt(Edge edge) +{ + if (!needsTypeCheck(edge, SpecInt32 | SpecInt52AsDouble)) + return; + + GPRTemporary temp(this); + convertMachineInt(edge, temp.gpr()); +} + +void SpeculativeJIT::speculateDoubleRepMachineInt(Edge edge) +{ + if (!needsTypeCheck(edge, SpecInt52AsDouble)) + return; + + SpeculateDoubleOperand value(this, edge); + FPRReg valueFPR = value.fpr(); + + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + + callOperation(operationConvertDoubleToInt52, resultGPR, valueFPR); + + DFG_TYPE_CHECK( + JSValueRegs(), edge, SpecInt52AsDouble, + m_jit.branch64( + JITCompiler::Equal, resultGPR, + JITCompiler::TrustedImm64(JSValue::notInt52))); +} + #endif } } // namespace JSC::DFG