X-Git-Url: https://git.saurik.com/apple/javascriptcore.git/blobdiff_plain/b80e619319b1def83d1e8b4f84042b661be1be7f..8b637bb680022adfddad653280734877951535a9:/assembler/MacroAssemblerARMv7.h diff --git a/assembler/MacroAssemblerARMv7.h b/assembler/MacroAssemblerARMv7.h index e3e928d..9cbb71c 100644 --- a/assembler/MacroAssemblerARMv7.h +++ b/assembler/MacroAssemblerARMv7.h @@ -45,18 +45,25 @@ class MacroAssemblerARMv7 : public AbstractMacroAssembler { inline ARMRegisters::FPSingleRegisterID fpTempRegisterAsSingle() { return ARMRegisters::asSingle(fpTempRegister); } public: + MacroAssemblerARMv7() + : m_makeJumpPatchable(false) + { + } + typedef ARMv7Assembler::LinkRecord LinkRecord; typedef ARMv7Assembler::JumpType JumpType; typedef ARMv7Assembler::JumpLinkType JumpLinkType; + typedef ARMv7Assembler::Condition Condition; - MacroAssemblerARMv7() - : m_inUninterruptedSequence(false) + static const ARMv7Assembler::Condition DefaultCondition = ARMv7Assembler::ConditionInvalid; + static const ARMv7Assembler::JumpType DefaultJump = ARMv7Assembler::JumpNoConditionFixedSize; + + static bool isCompactPtrAlignedAddressOffset(ptrdiff_t value) { + return value >= -255 && value <= 255; } - - void beginUninterruptedSequence() { m_inUninterruptedSequence = true; } - void endUninterruptedSequence() { m_inUninterruptedSequence = false; } - Vector& jumpsToLink() { return m_assembler.jumpsToLink(); } + + Vector& jumpsToLink() { return m_assembler.jumpsToLink(); } void* unlinkedCode() { return m_assembler.unlinkedCode(); } bool canCompact(JumpType jumpType) { return m_assembler.canCompact(jumpType); } JumpLinkType computeJumpType(JumpType jumpType, const uint8_t* from, const uint8_t* to) { return m_assembler.computeJumpType(jumpType, from, to); } @@ -100,7 +107,7 @@ public: static const Scale ScalePtr = TimesFour; - enum Condition { + enum RelationalCondition { Equal = ARMv7Assembler::ConditionEQ, NotEqual = ARMv7Assembler::ConditionNE, Above = ARMv7Assembler::ConditionHI, @@ -110,12 +117,17 @@ public: GreaterThan = ARMv7Assembler::ConditionGT, GreaterThanOrEqual = ARMv7Assembler::ConditionGE, LessThan = ARMv7Assembler::ConditionLT, - LessThanOrEqual = ARMv7Assembler::ConditionLE, + LessThanOrEqual = ARMv7Assembler::ConditionLE + }; + + enum ResultCondition { Overflow = ARMv7Assembler::ConditionVS, Signed = ARMv7Assembler::ConditionMI, + PositiveOrZero = ARMv7Assembler::ConditionPL, Zero = ARMv7Assembler::ConditionEQ, NonZero = ARMv7Assembler::ConditionNE }; + enum DoubleCondition { // These conditions will only evaluate to true if the comparison is ordered - i.e. neither operand is NaN. DoubleEqual = ARMv7Assembler::ConditionEQ, @@ -139,7 +151,7 @@ public: // Integer arithmetic operations: // // Operations are typically two operand - operation(source, srcDst) - // For many operations the source may be an Imm32, the srcDst operand + // For many operations the source may be an TrustedImm32, the srcDst operand // may often be a memory location (explictly described using an Address // object). @@ -148,12 +160,18 @@ public: m_assembler.add(dest, dest, src); } - void add32(Imm32 imm, RegisterID dest) + void add32(TrustedImm32 imm, RegisterID dest) { add32(imm, dest, dest); } + + void add32(AbsoluteAddress src, RegisterID dest) + { + load32(src.m_ptr, dataTempRegister); + add32(dataTempRegister, dest); + } - void add32(Imm32 imm, RegisterID src, RegisterID dest) + void add32(TrustedImm32 imm, RegisterID src, RegisterID dest) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12OrEncodedImm(imm.m_value); if (armImm.isValid()) @@ -164,7 +182,7 @@ public: } } - void add32(Imm32 imm, Address address) + void add32(TrustedImm32 imm, Address address) { load32(address, dataTempRegister); @@ -187,7 +205,7 @@ public: add32(dataTempRegister, dest); } - void add32(Imm32 imm, AbsoluteAddress address) + void add32(TrustedImm32 imm, AbsoluteAddress address) { load32(address.m_ptr, dataTempRegister); @@ -204,35 +222,86 @@ public: store32(dataTempRegister, address.m_ptr); } - void and32(RegisterID src, RegisterID dest) + void add64(TrustedImm32 imm, AbsoluteAddress address) { - m_assembler.ARM_and(dest, dest, src); + move(TrustedImmPtr(address.m_ptr), addressTempRegister); + + m_assembler.ldr(dataTempRegister, addressTempRegister, ARMThumbImmediate::makeUInt12(0)); + ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); + if (armImm.isValid()) + m_assembler.add_S(dataTempRegister, dataTempRegister, armImm); + else { + move(imm, addressTempRegister); + m_assembler.add_S(dataTempRegister, dataTempRegister, addressTempRegister); + move(TrustedImmPtr(address.m_ptr), addressTempRegister); + } + m_assembler.str(dataTempRegister, addressTempRegister, ARMThumbImmediate::makeUInt12(0)); + + m_assembler.ldr(dataTempRegister, addressTempRegister, ARMThumbImmediate::makeUInt12(4)); + m_assembler.adc(dataTempRegister, dataTempRegister, ARMThumbImmediate::makeEncodedImm(imm.m_value >> 31)); + m_assembler.str(dataTempRegister, addressTempRegister, ARMThumbImmediate::makeUInt12(4)); } - void and32(Imm32 imm, RegisterID dest) + void and32(RegisterID op1, RegisterID op2, RegisterID dest) + { + m_assembler.ARM_and(dest, op1, op2); + } + + void and32(TrustedImm32 imm, RegisterID src, RegisterID dest) { ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); if (armImm.isValid()) - m_assembler.ARM_and(dest, dest, armImm); + m_assembler.ARM_and(dest, src, armImm); else { move(imm, dataTempRegister); - m_assembler.ARM_and(dest, dest, dataTempRegister); + m_assembler.ARM_and(dest, src, dataTempRegister); } } - void lshift32(RegisterID shift_amount, RegisterID dest) + void and32(RegisterID src, RegisterID dest) + { + and32(dest, src, dest); + } + + void and32(TrustedImm32 imm, RegisterID dest) + { + and32(imm, dest, dest); + } + + void and32(Address src, RegisterID dest) + { + load32(src, dataTempRegister); + and32(dataTempRegister, dest); + } + + void countLeadingZeros32(RegisterID src, RegisterID dest) + { + m_assembler.clz(dest, src); + } + + void lshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { // Clamp the shift to the range 0..31 ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(0x1f); ASSERT(armImm.isValid()); - m_assembler.ARM_and(dataTempRegister, shift_amount, armImm); + m_assembler.ARM_and(dataTempRegister, shiftAmount, armImm); + + m_assembler.lsl(dest, src, dataTempRegister); + } + + void lshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) + { + m_assembler.lsl(dest, src, imm.m_value & 0x1f); + } - m_assembler.lsl(dest, dest, dataTempRegister); + void lshift32(RegisterID shiftAmount, RegisterID dest) + { + lshift32(dest, shiftAmount, dest); } - void lshift32(Imm32 imm, RegisterID dest) + void lshift32(TrustedImm32 imm, RegisterID dest) { - m_assembler.lsl(dest, dest, imm.m_value & 0x1f); + lshift32(dest, imm, dest); } void mul32(RegisterID src, RegisterID dest) @@ -240,7 +309,7 @@ public: m_assembler.smull(dest, dataTempRegister, dest, src); } - void mul32(Imm32 imm, RegisterID src, RegisterID dest) + void mul32(TrustedImm32 imm, RegisterID src, RegisterID dest) { move(imm, dataTempRegister); m_assembler.smull(dest, dataTempRegister, src, dataTempRegister); @@ -251,55 +320,88 @@ public: m_assembler.neg(srcDest, srcDest); } - void not32(RegisterID srcDest) + void or32(RegisterID src, RegisterID dest) + { + m_assembler.orr(dest, dest, src); + } + + void or32(RegisterID src, AbsoluteAddress dest) + { + move(TrustedImmPtr(dest.m_ptr), addressTempRegister); + load32(addressTempRegister, dataTempRegister); + or32(src, dataTempRegister); + store32(dataTempRegister, addressTempRegister); + } + + void or32(TrustedImm32 imm, RegisterID dest) { - m_assembler.mvn(srcDest, srcDest); + or32(imm, dest, dest); } - void or32(RegisterID src, RegisterID dest) + void or32(RegisterID op1, RegisterID op2, RegisterID dest) { - m_assembler.orr(dest, dest, src); + m_assembler.orr(dest, op1, op2); } - void or32(Imm32 imm, RegisterID dest) + void or32(TrustedImm32 imm, RegisterID src, RegisterID dest) { ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); if (armImm.isValid()) - m_assembler.orr(dest, dest, armImm); + m_assembler.orr(dest, src, armImm); else { move(imm, dataTempRegister); - m_assembler.orr(dest, dest, dataTempRegister); + m_assembler.orr(dest, src, dataTempRegister); } } - void rshift32(RegisterID shift_amount, RegisterID dest) + void rshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { // Clamp the shift to the range 0..31 ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(0x1f); ASSERT(armImm.isValid()); - m_assembler.ARM_and(dataTempRegister, shift_amount, armImm); + m_assembler.ARM_and(dataTempRegister, shiftAmount, armImm); - m_assembler.asr(dest, dest, dataTempRegister); + m_assembler.asr(dest, src, dataTempRegister); } - void rshift32(Imm32 imm, RegisterID dest) + void rshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { - m_assembler.asr(dest, dest, imm.m_value & 0x1f); + m_assembler.asr(dest, src, imm.m_value & 0x1f); + } + + void rshift32(RegisterID shiftAmount, RegisterID dest) + { + rshift32(dest, shiftAmount, dest); } - void urshift32(RegisterID shift_amount, RegisterID dest) + void rshift32(TrustedImm32 imm, RegisterID dest) + { + rshift32(dest, imm, dest); + } + + void urshift32(RegisterID src, RegisterID shiftAmount, RegisterID dest) { // Clamp the shift to the range 0..31 ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(0x1f); ASSERT(armImm.isValid()); - m_assembler.ARM_and(dataTempRegister, shift_amount, armImm); + m_assembler.ARM_and(dataTempRegister, shiftAmount, armImm); - m_assembler.lsr(dest, dest, dataTempRegister); + m_assembler.lsr(dest, src, dataTempRegister); } - void urshift32(Imm32 imm, RegisterID dest) + void urshift32(RegisterID src, TrustedImm32 imm, RegisterID dest) { - m_assembler.lsr(dest, dest, imm.m_value & 0x1f); + m_assembler.lsr(dest, src, imm.m_value & 0x1f); + } + + void urshift32(RegisterID shiftAmount, RegisterID dest) + { + urshift32(dest, shiftAmount, dest); + } + + void urshift32(TrustedImm32 imm, RegisterID dest) + { + urshift32(dest, imm, dest); } void sub32(RegisterID src, RegisterID dest) @@ -307,7 +409,7 @@ public: m_assembler.sub(dest, dest, src); } - void sub32(Imm32 imm, RegisterID dest) + void sub32(TrustedImm32 imm, RegisterID dest) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12OrEncodedImm(imm.m_value); if (armImm.isValid()) @@ -318,7 +420,7 @@ public: } } - void sub32(Imm32 imm, Address address) + void sub32(TrustedImm32 imm, Address address) { load32(address, dataTempRegister); @@ -341,7 +443,7 @@ public: sub32(dataTempRegister, dest); } - void sub32(Imm32 imm, AbsoluteAddress address) + void sub32(TrustedImm32 imm, AbsoluteAddress address) { load32(address.m_ptr, dataTempRegister); @@ -358,27 +460,45 @@ public: store32(dataTempRegister, address.m_ptr); } - void xor32(RegisterID src, RegisterID dest) + void xor32(RegisterID op1, RegisterID op2, RegisterID dest) { - m_assembler.eor(dest, dest, src); + m_assembler.eor(dest, op1, op2); } - void xor32(Imm32 imm, RegisterID dest) + void xor32(TrustedImm32 imm, RegisterID src, RegisterID dest) { + if (imm.m_value == -1) { + m_assembler.mvn(dest, src); + return; + } + ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); if (armImm.isValid()) - m_assembler.eor(dest, dest, armImm); + m_assembler.eor(dest, src, armImm); else { move(imm, dataTempRegister); - m_assembler.eor(dest, dest, dataTempRegister); + m_assembler.eor(dest, src, dataTempRegister); } } + + void xor32(RegisterID src, RegisterID dest) + { + xor32(dest, src, dest); + } + + void xor32(TrustedImm32 imm, RegisterID dest) + { + if (imm.m_value == -1) + m_assembler.mvn(dest, dest); + else + xor32(imm, dest, dest); + } // Memory access operations: // // Loads are of the form load(address, destination) and stores of the form - // store(source, address). The source for a store may be an Imm32. Address + // store(source, address). The source for a store may be an TrustedImm32. Address // operand objects to loads and store will be implicitly constructed if a // register is passed. @@ -410,6 +530,12 @@ private: m_assembler.ldrh(dest, address.base, address.u.offset, true, false); } } + + void load16Signed(ArmAddress address, RegisterID dest) + { + ASSERT(address.type == ArmAddress::HasIndex); + m_assembler.ldrsh(dest, address.base, address.u.index, address.u.scale); + } void load8(ArmAddress address, RegisterID dest) { @@ -424,7 +550,14 @@ private: m_assembler.ldrb(dest, address.base, address.u.offset, true, false); } } + + void load8Signed(ArmAddress address, RegisterID dest) + { + ASSERT(address.type == ArmAddress::HasIndex); + m_assembler.ldrsb(dest, address.base, address.u.index, address.u.scale); + } +protected: void store32(RegisterID src, ArmAddress address) { if (address.type == ArmAddress::HasIndex) @@ -439,6 +572,35 @@ private: } } +private: + void store8(RegisterID src, ArmAddress address) + { + if (address.type == ArmAddress::HasIndex) + m_assembler.strb(src, address.base, address.u.index, address.u.scale); + else if (address.u.offset >= 0) { + ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); + ASSERT(armImm.isValid()); + m_assembler.strb(src, address.base, armImm); + } else { + ASSERT(address.u.offset >= -255); + m_assembler.strb(src, address.base, address.u.offset, true, false); + } + } + + void store16(RegisterID src, ArmAddress address) + { + if (address.type == ArmAddress::HasIndex) + m_assembler.strh(src, address.base, address.u.index, address.u.scale); + else if (address.u.offset >= 0) { + ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.u.offset); + ASSERT(armImm.isValid()); + m_assembler.strh(src, address.base, armImm); + } else { + ASSERT(address.u.offset >= -255); + m_assembler.strh(src, address.base, address.u.offset, true, false); + } + } + public: void load32(ImplicitAddress address, RegisterID dest) { @@ -455,29 +617,62 @@ public: load32(setupArmAddress(address), dest); } - void load32(void* address, RegisterID dest) + void load16Unaligned(BaseIndex address, RegisterID dest) + { + load16(setupArmAddress(address), dest); + } + + void load32(const void* address, RegisterID dest) { - move(ImmPtr(address), addressTempRegister); + move(TrustedImmPtr(address), addressTempRegister); m_assembler.ldr(dest, addressTempRegister, ARMThumbImmediate::makeUInt16(0)); } + + ConvertibleLoadLabel convertibleLoadPtr(Address address, RegisterID dest) + { + ConvertibleLoadLabel result(this); + ASSERT(address.offset >= 0 && address.offset <= 255); + m_assembler.ldrWide8BitImmediate(dest, address.base, address.offset); + return result; + } void load8(ImplicitAddress address, RegisterID dest) { load8(setupArmAddress(address), dest); } + void load8Signed(ImplicitAddress, RegisterID) + { + UNREACHABLE_FOR_PLATFORM(); + } + + void load8(BaseIndex address, RegisterID dest) + { + load8(setupArmAddress(address), dest); + } + + void load8Signed(BaseIndex address, RegisterID dest) + { + load8Signed(setupArmAddress(address), dest); + } + DataLabel32 load32WithAddressOffsetPatch(Address address, RegisterID dest) { - DataLabel32 label = moveWithPatch(Imm32(address.offset), dataTempRegister); + DataLabel32 label = moveWithPatch(TrustedImm32(address.offset), dataTempRegister); load32(ArmAddress(address.base, dataTempRegister), dest); return label; } - - Label loadPtrWithPatchToLEA(Address address, RegisterID dest) + + DataLabelCompact load32WithCompactAddressOffsetPatch(Address address, RegisterID dest) { - Label label(this); - moveFixedWidthEncoding(Imm32(address.offset), dataTempRegister); - load32(ArmAddress(address.base, dataTempRegister), dest); + padBeforePatch(); + + RegisterID base = address.base; + + DataLabelCompact label(this); + ASSERT(isCompactPtrAlignedAddressOffset(address.offset)); + + m_assembler.ldr(dest, base, address.offset, true, false); return label; } @@ -486,20 +681,30 @@ public: m_assembler.ldrh(dest, makeBaseIndexBase(address), address.index, address.scale); } + void load16Signed(BaseIndex address, RegisterID dest) + { + load16Signed(setupArmAddress(address), dest); + } + void load16(ImplicitAddress address, RegisterID dest) { ARMThumbImmediate armImm = ARMThumbImmediate::makeUInt12(address.offset); if (armImm.isValid()) m_assembler.ldrh(dest, address.base, armImm); else { - move(Imm32(address.offset), dataTempRegister); + move(TrustedImm32(address.offset), dataTempRegister); m_assembler.ldrh(dest, address.base, dataTempRegister); } } + + void load16Signed(ImplicitAddress, RegisterID) + { + UNREACHABLE_FOR_PLATFORM(); + } DataLabel32 store32WithAddressOffsetPatch(RegisterID src, Address address) { - DataLabel32 label = moveWithPatch(Imm32(address.offset), dataTempRegister); + DataLabel32 label = moveWithPatch(TrustedImm32(address.offset), dataTempRegister); store32(src, ArmAddress(address.base, dataTempRegister)); return label; } @@ -514,45 +719,90 @@ public: store32(src, setupArmAddress(address)); } - void store32(Imm32 imm, ImplicitAddress address) + void store32(TrustedImm32 imm, ImplicitAddress address) + { + move(imm, dataTempRegister); + store32(dataTempRegister, setupArmAddress(address)); + } + + void store32(TrustedImm32 imm, BaseIndex address) { move(imm, dataTempRegister); store32(dataTempRegister, setupArmAddress(address)); } - void store32(RegisterID src, void* address) + void store32(RegisterID src, const void* address) { - move(ImmPtr(address), addressTempRegister); + move(TrustedImmPtr(address), addressTempRegister); m_assembler.str(src, addressTempRegister, ARMThumbImmediate::makeUInt16(0)); } - void store32(Imm32 imm, void* address) + void store32(TrustedImm32 imm, const void* address) { move(imm, dataTempRegister); store32(dataTempRegister, address); } + void store8(RegisterID src, BaseIndex address) + { + store8(src, setupArmAddress(address)); + } + + void store8(RegisterID src, void* address) + { + move(TrustedImmPtr(address), addressTempRegister); + store8(src, ArmAddress(addressTempRegister, 0)); + } + + void store8(TrustedImm32 imm, void* address) + { + move(imm, dataTempRegister); + store8(dataTempRegister, address); + } + + void store16(RegisterID src, BaseIndex address) + { + store16(src, setupArmAddress(address)); + } - // Floating-point operations: - - bool supportsFloatingPoint() const { return true; } - // On x86(_64) the MacroAssembler provides an interface to truncate a double to an integer. - // If a value is not representable as an integer, and possibly for some values that are, - // (on x86 INT_MIN, since this is indistinguishable from results for out-of-range/NaN input) - // a branch will be taken. It is not clear whether this interface will be well suited to - // other platforms. On ARMv7 the hardware truncation operation produces multiple possible - // failure values (saturates to INT_MIN & INT_MAX, NaN reulsts in a value of 0). This is a - // temporary solution while we work out what this interface should be. Either we need to - // decide to make this interface work on all platforms, rework the interface to make it more - // generic, or decide that the MacroAssembler cannot practically be used to abstracted these - // operations, and make clients go directly to the m_assembler to plant truncation instructions. - // In short, FIXME:. - bool supportsFloatingPointTruncate() const { return false; } + // Possibly clobbers src, but not on this architecture. + void moveDoubleToInts(FPRegisterID src, RegisterID dest1, RegisterID dest2) + { + m_assembler.vmov(dest1, dest2, src); + } + + void moveIntsToDouble(RegisterID src1, RegisterID src2, FPRegisterID dest, FPRegisterID scratch) + { + UNUSED_PARAM(scratch); + m_assembler.vmov(dest, src1, src2); + } - bool supportsFloatingPointSqrt() const +#if ENABLE(JIT_CONSTANT_BLINDING) + static bool shouldBlindForSpecificArch(uint32_t value) { - return false; + ARMThumbImmediate immediate = ARMThumbImmediate::makeEncodedImm(value); + + // Couldn't be encoded as an immediate, so assume it's untrusted. + if (!immediate.isValid()) + return true; + + // If we can encode the immediate, we have less than 16 attacker + // controlled bits. + if (immediate.isEncodedImm()) + return false; + + // Don't let any more than 12 bits of an instruction word + // be controlled by an attacker. + return !immediate.isUInt12(); } +#endif + + // Floating-point operations: + + static bool supportsFloatingPoint() { return true; } + static bool supportsFloatingPointTruncate() { return true; } + static bool supportsFloatingPointSqrt() { return true; } + static bool supportsFloatingPointAbs() { return true; } void loadDouble(ImplicitAddress address, FPRegisterID dest) { @@ -561,7 +811,7 @@ public: // Arm vfp addresses can be offset by a 9-bit ones-comp immediate, left shifted by 2. if ((offset & 3) || (offset > (255 * 4)) || (offset < -(255 * 4))) { - add32(Imm32(offset), base, addressTempRegister); + add32(TrustedImm32(offset), base, addressTempRegister); base = addressTempRegister; offset = 0; } @@ -569,9 +819,46 @@ public: m_assembler.vldr(dest, base, offset); } + void loadFloat(ImplicitAddress address, FPRegisterID dest) + { + RegisterID base = address.base; + int32_t offset = address.offset; + + // Arm vfp addresses can be offset by a 9-bit ones-comp immediate, left shifted by 2. + if ((offset & 3) || (offset > (255 * 4)) || (offset < -(255 * 4))) { + add32(TrustedImm32(offset), base, addressTempRegister); + base = addressTempRegister; + offset = 0; + } + + m_assembler.flds(ARMRegisters::asSingle(dest), base, offset); + } + + void loadDouble(BaseIndex address, FPRegisterID dest) + { + move(address.index, addressTempRegister); + lshift32(TrustedImm32(address.scale), addressTempRegister); + add32(address.base, addressTempRegister); + loadDouble(Address(addressTempRegister, address.offset), dest); + } + + void loadFloat(BaseIndex address, FPRegisterID dest) + { + move(address.index, addressTempRegister); + lshift32(TrustedImm32(address.scale), addressTempRegister); + add32(address.base, addressTempRegister); + loadFloat(Address(addressTempRegister, address.offset), dest); + } + + void moveDouble(FPRegisterID src, FPRegisterID dest) + { + if (src != dest) + m_assembler.vmov(dest, src); + } + void loadDouble(const void* address, FPRegisterID dest) { - move(ImmPtr(address), addressTempRegister); + move(TrustedImmPtr(address), addressTempRegister); m_assembler.vldr(dest, addressTempRegister, 0); } @@ -582,7 +869,7 @@ public: // Arm vfp addresses can be offset by a 9-bit ones-comp immediate, left shifted by 2. if ((offset & 3) || (offset > (255 * 4)) || (offset < -(255 * 4))) { - add32(Imm32(offset), base, addressTempRegister); + add32(TrustedImm32(offset), base, addressTempRegister); base = addressTempRegister; offset = 0; } @@ -590,9 +877,46 @@ public: m_assembler.vstr(src, base, offset); } + void storeFloat(FPRegisterID src, ImplicitAddress address) + { + RegisterID base = address.base; + int32_t offset = address.offset; + + // Arm vfp addresses can be offset by a 9-bit ones-comp immediate, left shifted by 2. + if ((offset & 3) || (offset > (255 * 4)) || (offset < -(255 * 4))) { + add32(TrustedImm32(offset), base, addressTempRegister); + base = addressTempRegister; + offset = 0; + } + + m_assembler.fsts(ARMRegisters::asSingle(src), base, offset); + } + + void storeDouble(FPRegisterID src, const void* address) + { + move(TrustedImmPtr(address), addressTempRegister); + storeDouble(src, addressTempRegister); + } + + void storeDouble(FPRegisterID src, BaseIndex address) + { + move(address.index, addressTempRegister); + lshift32(TrustedImm32(address.scale), addressTempRegister); + add32(address.base, addressTempRegister); + storeDouble(src, Address(addressTempRegister, address.offset)); + } + + void storeFloat(FPRegisterID src, BaseIndex address) + { + move(address.index, addressTempRegister); + lshift32(TrustedImm32(address.scale), addressTempRegister); + add32(address.base, addressTempRegister); + storeFloat(src, Address(addressTempRegister, address.offset)); + } + void addDouble(FPRegisterID src, FPRegisterID dest) { - m_assembler.vadd_F64(dest, dest, src); + m_assembler.vadd(dest, dest, src); } void addDouble(Address src, FPRegisterID dest) @@ -601,14 +925,30 @@ public: addDouble(fpTempRegister, dest); } + void addDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) + { + m_assembler.vadd(dest, op1, op2); + } + + void addDouble(AbsoluteAddress address, FPRegisterID dest) + { + loadDouble(address.m_ptr, fpTempRegister); + m_assembler.vadd(dest, dest, fpTempRegister); + } + void divDouble(FPRegisterID src, FPRegisterID dest) { - m_assembler.vdiv_F64(dest, dest, src); + m_assembler.vdiv(dest, dest, src); + } + + void divDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) + { + m_assembler.vdiv(dest, op1, op2); } void subDouble(FPRegisterID src, FPRegisterID dest) { - m_assembler.vsub_F64(dest, dest, src); + m_assembler.vsub(dest, dest, src); } void subDouble(Address src, FPRegisterID dest) @@ -617,9 +957,14 @@ public: subDouble(fpTempRegister, dest); } + void subDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) + { + m_assembler.vsub(dest, op1, op2); + } + void mulDouble(FPRegisterID src, FPRegisterID dest) { - m_assembler.vmul_F64(dest, dest, src); + m_assembler.vmul(dest, dest, src); } void mulDouble(Address src, FPRegisterID dest) @@ -628,36 +973,61 @@ public: mulDouble(fpTempRegister, dest); } - void sqrtDouble(FPRegisterID, FPRegisterID) + void mulDouble(FPRegisterID op1, FPRegisterID op2, FPRegisterID dest) + { + m_assembler.vmul(dest, op1, op2); + } + + void sqrtDouble(FPRegisterID src, FPRegisterID dest) + { + m_assembler.vsqrt(dest, src); + } + + void absDouble(FPRegisterID src, FPRegisterID dest) + { + m_assembler.vabs(dest, src); + } + + void negateDouble(FPRegisterID src, FPRegisterID dest) { - ASSERT_NOT_REACHED(); + m_assembler.vneg(dest, src); } void convertInt32ToDouble(RegisterID src, FPRegisterID dest) { - m_assembler.vmov(fpTempRegisterAsSingle(), src); - m_assembler.vcvt_F64_S32(dest, fpTempRegisterAsSingle()); + m_assembler.vmov(fpTempRegister, src, src); + m_assembler.vcvt_signedToFloatingPoint(dest, fpTempRegisterAsSingle()); } void convertInt32ToDouble(Address address, FPRegisterID dest) { // Fixme: load directly into the fpr! load32(address, dataTempRegister); - m_assembler.vmov(fpTempRegisterAsSingle(), dataTempRegister); - m_assembler.vcvt_F64_S32(dest, fpTempRegisterAsSingle()); + m_assembler.vmov(fpTempRegister, dataTempRegister, dataTempRegister); + m_assembler.vcvt_signedToFloatingPoint(dest, fpTempRegisterAsSingle()); } void convertInt32ToDouble(AbsoluteAddress address, FPRegisterID dest) { // Fixme: load directly into the fpr! load32(address.m_ptr, dataTempRegister); - m_assembler.vmov(fpTempRegisterAsSingle(), dataTempRegister); - m_assembler.vcvt_F64_S32(dest, fpTempRegisterAsSingle()); + m_assembler.vmov(fpTempRegister, dataTempRegister, dataTempRegister); + m_assembler.vcvt_signedToFloatingPoint(dest, fpTempRegisterAsSingle()); + } + + void convertFloatToDouble(FPRegisterID src, FPRegisterID dst) + { + m_assembler.vcvtds(dst, ARMRegisters::asSingle(src)); + } + + void convertDoubleToFloat(FPRegisterID src, FPRegisterID dst) + { + m_assembler.vcvtsd(ARMRegisters::asSingle(dst), src); } Jump branchDouble(DoubleCondition cond, FPRegisterID left, FPRegisterID right) { - m_assembler.vcmp_F64(left, right); + m_assembler.vcmp(left, right); m_assembler.vmrs(); if (cond == DoubleNotEqual) { @@ -672,39 +1042,89 @@ public: Jump notEqual = makeBranch(ARMv7Assembler::ConditionNE); unordered.link(this); // We get here if either unordered or equal. - Jump result = makeJump(); + Jump result = jump(); notEqual.link(this); return result; } return makeBranch(cond); } - Jump branchTruncateDoubleToInt32(FPRegisterID, RegisterID) + enum BranchTruncateType { BranchIfTruncateFailed, BranchIfTruncateSuccessful }; + Jump branchTruncateDoubleToInt32(FPRegisterID src, RegisterID dest, BranchTruncateType branchType = BranchIfTruncateFailed) + { + // Convert into dest. + m_assembler.vcvt_floatingPointToSigned(fpTempRegisterAsSingle(), src); + m_assembler.vmov(dest, fpTempRegisterAsSingle()); + + // Calculate 2x dest. If the value potentially underflowed, it will have + // clamped to 0x80000000, so 2x dest is zero in this case. In the case of + // overflow the result will be equal to -2. + Jump underflow = branchAdd32(Zero, dest, dest, dataTempRegister); + Jump noOverflow = branch32(NotEqual, dataTempRegister, TrustedImm32(-2)); + + // For BranchIfTruncateSuccessful, we branch if 'noOverflow' jumps. + underflow.link(this); + if (branchType == BranchIfTruncateSuccessful) + return noOverflow; + + // We'll reach the current point in the code on failure, so plant a + // jump here & link the success case. + Jump failure = jump(); + noOverflow.link(this); + return failure; + } + + Jump branchTruncateDoubleToUint32(FPRegisterID src, RegisterID dest, BranchTruncateType branchType = BranchIfTruncateFailed) + { + m_assembler.vcvt_floatingPointToSigned(fpTempRegisterAsSingle(), src); + m_assembler.vmov(dest, fpTempRegisterAsSingle()); + + Jump overflow = branch32(Equal, dest, TrustedImm32(0x7fffffff)); + Jump success = branch32(GreaterThanOrEqual, dest, TrustedImm32(0)); + overflow.link(this); + + if (branchType == BranchIfTruncateSuccessful) + return success; + + Jump failure = jump(); + success.link(this); + return failure; + } + + // Result is undefined if the value is outside of the integer range. + void truncateDoubleToInt32(FPRegisterID src, RegisterID dest) { - ASSERT_NOT_REACHED(); - return jump(); + m_assembler.vcvt_floatingPointToSigned(fpTempRegisterAsSingle(), src); + m_assembler.vmov(dest, fpTempRegisterAsSingle()); } + void truncateDoubleToUint32(FPRegisterID src, RegisterID dest) + { + m_assembler.vcvt_floatingPointToUnsigned(fpTempRegisterAsSingle(), src); + m_assembler.vmov(dest, fpTempRegisterAsSingle()); + } + // Convert 'src' to an integer, and places the resulting 'dest'. // If the result is not representable as a 32 bit value, branch. // May also branch for some values that are representable in 32 bits // (specifically, in this case, 0). - void branchConvertDoubleToInt32(FPRegisterID src, RegisterID dest, JumpList& failureCases, FPRegisterID) + void branchConvertDoubleToInt32(FPRegisterID src, RegisterID dest, JumpList& failureCases, FPRegisterID, bool negZeroCheck = true) { - m_assembler.vcvtr_S32_F64(fpTempRegisterAsSingle(), src); + m_assembler.vcvt_floatingPointToSigned(fpTempRegisterAsSingle(), src); m_assembler.vmov(dest, fpTempRegisterAsSingle()); // Convert the integer result back to float & compare to the original value - if not equal or unordered (NaN) then jump. - m_assembler.vcvt_F64_S32(fpTempRegister, fpTempRegisterAsSingle()); + m_assembler.vcvt_signedToFloatingPoint(fpTempRegister, fpTempRegisterAsSingle()); failureCases.append(branchDouble(DoubleNotEqualOrUnordered, src, fpTempRegister)); // If the result is zero, it might have been -0.0, and the double comparison won't catch this! - failureCases.append(branchTest32(Zero, dest)); + if (negZeroCheck) + failureCases.append(branchTest32(Zero, dest)); } Jump branchDoubleNonZero(FPRegisterID reg, FPRegisterID) { - m_assembler.vcmpz_F64(reg); + m_assembler.vcmpz(reg); m_assembler.vmrs(); Jump unordered = makeBranch(ARMv7Assembler::ConditionVS); Jump result = makeBranch(ARMv7Assembler::ConditionNE); @@ -714,13 +1134,13 @@ public: Jump branchDoubleZeroOrNaN(FPRegisterID reg, FPRegisterID) { - m_assembler.vcmpz_F64(reg); + m_assembler.vcmpz(reg); m_assembler.vmrs(); Jump unordered = makeBranch(ARMv7Assembler::ConditionVS); Jump notEqual = makeBranch(ARMv7Assembler::ConditionNE); unordered.link(this); // We get here if either unordered or equal. - Jump result = makeJump(); + Jump result = jump(); notEqual.link(this); return result; } @@ -751,7 +1171,7 @@ public: push(dataTempRegister); } - void push(Imm32 imm) + void push(TrustedImm32 imm) { move(imm, dataTempRegister); push(dataTempRegister); @@ -761,35 +1181,32 @@ public: // // Move values in registers. - void move(Imm32 imm, RegisterID dest) + void move(TrustedImm32 imm, RegisterID dest) { uint32_t value = imm.m_value; - if (imm.m_isPointer) - moveFixedWidthEncoding(imm, dest); - else { - ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(value); + ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(value); - if (armImm.isValid()) - m_assembler.mov(dest, armImm); - else if ((armImm = ARMThumbImmediate::makeEncodedImm(~value)).isValid()) - m_assembler.mvn(dest, armImm); - else { - m_assembler.mov(dest, ARMThumbImmediate::makeUInt16(value)); - if (value & 0xffff0000) - m_assembler.movt(dest, ARMThumbImmediate::makeUInt16(value >> 16)); - } + if (armImm.isValid()) + m_assembler.mov(dest, armImm); + else if ((armImm = ARMThumbImmediate::makeEncodedImm(~value)).isValid()) + m_assembler.mvn(dest, armImm); + else { + m_assembler.mov(dest, ARMThumbImmediate::makeUInt16(value)); + if (value & 0xffff0000) + m_assembler.movt(dest, ARMThumbImmediate::makeUInt16(value >> 16)); } } void move(RegisterID src, RegisterID dest) { - m_assembler.mov(dest, src); + if (src != dest) + m_assembler.mov(dest, src); } - void move(ImmPtr imm, RegisterID dest) + void move(TrustedImmPtr imm, RegisterID dest) { - move(Imm32(imm), dest); + move(TrustedImm32(imm), dest); } void swap(RegisterID reg1, RegisterID reg2) @@ -801,16 +1218,34 @@ public: void signExtend32ToPtr(RegisterID src, RegisterID dest) { - if (src != dest) - move(src, dest); + move(src, dest); } void zeroExtend32ToPtr(RegisterID src, RegisterID dest) { - if (src != dest) - move(src, dest); + move(src, dest); } + // Invert a relational condition, e.g. == becomes !=, < becomes >=, etc. + static RelationalCondition invert(RelationalCondition cond) + { + return static_cast(cond ^ 1); + } + + void nop() + { + m_assembler.nop(); + } + + static void replaceWithJump(CodeLocationLabel instructionStart, CodeLocationLabel destination) + { + ARMv7Assembler::replaceWithJump(instructionStart.dataLocation(), destination.dataLocation()); + } + + static ptrdiff_t maxJumpReplacementSize() + { + return ARMv7Assembler::maxJumpReplacementSize(); + } // Forwards / external control flow operations: // @@ -824,7 +1259,7 @@ public: // used (representing the names 'below' and 'above'). // // Operands to the comparision are provided in the expected order, e.g. - // jle32(reg1, Imm32(5)) will branch if the value held in reg1, when + // jle32(reg1, TrustedImm32(5)) will branch if the value held in reg1, when // treated as a signed 32bit value, is less than or equal to 5. // // jz and jnz test whether the first operand is equal to zero, and take @@ -832,7 +1267,7 @@ public: private: // Should we be using TEQ for equal/not-equal? - void compare32(RegisterID left, Imm32 right) + void compare32(RegisterID left, TrustedImm32 right) { int32_t imm = right.m_value; if (!imm) @@ -844,13 +1279,13 @@ private: else if ((armImm = ARMThumbImmediate::makeEncodedImm(-imm)).isValid()) m_assembler.cmn(left, armImm); else { - move(Imm32(imm), dataTempRegister); + move(TrustedImm32(imm), dataTempRegister); m_assembler.cmp(left, dataTempRegister); } } } - void test32(RegisterID reg, Imm32 mask) + void test32(RegisterID reg, TrustedImm32 mask) { int32_t imm = mask.m_value; @@ -868,153 +1303,144 @@ private: } public: - Jump branch32(Condition cond, RegisterID left, RegisterID right) + Jump branch32(RelationalCondition cond, RegisterID left, RegisterID right) { m_assembler.cmp(left, right); return Jump(makeBranch(cond)); } - Jump branch32(Condition cond, RegisterID left, Imm32 right) + Jump branch32(RelationalCondition cond, RegisterID left, TrustedImm32 right) { compare32(left, right); return Jump(makeBranch(cond)); } - Jump branch32(Condition cond, RegisterID left, Address right) + Jump branch32(RelationalCondition cond, RegisterID left, Address right) { load32(right, dataTempRegister); return branch32(cond, left, dataTempRegister); } - Jump branch32(Condition cond, Address left, RegisterID right) + Jump branch32(RelationalCondition cond, Address left, RegisterID right) { load32(left, dataTempRegister); return branch32(cond, dataTempRegister, right); } - Jump branch32(Condition cond, Address left, Imm32 right) + Jump branch32(RelationalCondition cond, Address left, TrustedImm32 right) { // use addressTempRegister incase the branch32 we call uses dataTempRegister. :-/ load32(left, addressTempRegister); return branch32(cond, addressTempRegister, right); } - Jump branch32(Condition cond, BaseIndex left, Imm32 right) + Jump branch32(RelationalCondition cond, BaseIndex left, TrustedImm32 right) { // use addressTempRegister incase the branch32 we call uses dataTempRegister. :-/ load32(left, addressTempRegister); return branch32(cond, addressTempRegister, right); } - Jump branch32WithUnalignedHalfWords(Condition cond, BaseIndex left, Imm32 right) + Jump branch32WithUnalignedHalfWords(RelationalCondition cond, BaseIndex left, TrustedImm32 right) { // use addressTempRegister incase the branch32 we call uses dataTempRegister. :-/ load32WithUnalignedHalfWords(left, addressTempRegister); return branch32(cond, addressTempRegister, right); } - Jump branch32(Condition cond, AbsoluteAddress left, RegisterID right) + Jump branch32(RelationalCondition cond, AbsoluteAddress left, RegisterID right) { load32(left.m_ptr, dataTempRegister); return branch32(cond, dataTempRegister, right); } - Jump branch32(Condition cond, AbsoluteAddress left, Imm32 right) + Jump branch32(RelationalCondition cond, AbsoluteAddress left, TrustedImm32 right) { // use addressTempRegister incase the branch32 we call uses dataTempRegister. :-/ load32(left.m_ptr, addressTempRegister); return branch32(cond, addressTempRegister, right); } - Jump branch16(Condition cond, BaseIndex left, RegisterID right) - { - load16(left, dataTempRegister); - m_assembler.lsl(addressTempRegister, right, 16); - m_assembler.lsl(dataTempRegister, dataTempRegister, 16); - return branch32(cond, dataTempRegister, addressTempRegister); - } - - Jump branch16(Condition cond, BaseIndex left, Imm32 right) - { - // use addressTempRegister incase the branch32 we call uses dataTempRegister. :-/ - load16(left, addressTempRegister); - m_assembler.lsl(addressTempRegister, addressTempRegister, 16); - return branch32(cond, addressTempRegister, Imm32(right.m_value << 16)); - } - - Jump branch8(Condition cond, RegisterID left, Imm32 right) + Jump branch8(RelationalCondition cond, RegisterID left, TrustedImm32 right) { compare32(left, right); return Jump(makeBranch(cond)); } - Jump branch8(Condition cond, Address left, Imm32 right) + Jump branch8(RelationalCondition cond, Address left, TrustedImm32 right) { + ASSERT(!(0xffffff00 & right.m_value)); // use addressTempRegister incase the branch8 we call uses dataTempRegister. :-/ load8(left, addressTempRegister); return branch8(cond, addressTempRegister, right); } - Jump branchTest32(Condition cond, RegisterID reg, RegisterID mask) + Jump branch8(RelationalCondition cond, BaseIndex left, TrustedImm32 right) + { + ASSERT(!(0xffffff00 & right.m_value)); + // use addressTempRegister incase the branch32 we call uses dataTempRegister. :-/ + load8(left, addressTempRegister); + return branch32(cond, addressTempRegister, right); + } + + Jump branchTest32(ResultCondition cond, RegisterID reg, RegisterID mask) { - ASSERT((cond == Zero) || (cond == NonZero)); m_assembler.tst(reg, mask); return Jump(makeBranch(cond)); } - Jump branchTest32(Condition cond, RegisterID reg, Imm32 mask = Imm32(-1)) + Jump branchTest32(ResultCondition cond, RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) { - ASSERT((cond == Zero) || (cond == NonZero)); test32(reg, mask); return Jump(makeBranch(cond)); } - Jump branchTest32(Condition cond, Address address, Imm32 mask = Imm32(-1)) + Jump branchTest32(ResultCondition cond, Address address, TrustedImm32 mask = TrustedImm32(-1)) { - ASSERT((cond == Zero) || (cond == NonZero)); // use addressTempRegister incase the branchTest32 we call uses dataTempRegister. :-/ load32(address, addressTempRegister); return branchTest32(cond, addressTempRegister, mask); } - Jump branchTest32(Condition cond, BaseIndex address, Imm32 mask = Imm32(-1)) + Jump branchTest32(ResultCondition cond, BaseIndex address, TrustedImm32 mask = TrustedImm32(-1)) { - ASSERT((cond == Zero) || (cond == NonZero)); // use addressTempRegister incase the branchTest32 we call uses dataTempRegister. :-/ load32(address, addressTempRegister); return branchTest32(cond, addressTempRegister, mask); } - Jump branchTest8(Condition cond, RegisterID reg, Imm32 mask = Imm32(-1)) - { - ASSERT((cond == Zero) || (cond == NonZero)); - test32(reg, mask); - return Jump(makeBranch(cond)); - } - - Jump branchTest8(Condition cond, Address address, Imm32 mask = Imm32(-1)) + Jump branchTest8(ResultCondition cond, Address address, TrustedImm32 mask = TrustedImm32(-1)) { - ASSERT((cond == Zero) || (cond == NonZero)); // use addressTempRegister incase the branchTest8 we call uses dataTempRegister. :-/ load8(address, addressTempRegister); - return branchTest8(cond, addressTempRegister, mask); + return branchTest32(cond, addressTempRegister, mask); } - Jump jump() + Jump branchTest8(ResultCondition cond, AbsoluteAddress address, TrustedImm32 mask = TrustedImm32(-1)) { - return Jump(makeJump()); + // use addressTempRegister incase the branchTest8 we call uses dataTempRegister. :-/ + move(TrustedImmPtr(address.m_ptr), addressTempRegister); + load8(Address(addressTempRegister), addressTempRegister); + return branchTest32(cond, addressTempRegister, mask); } void jump(RegisterID target) { - m_assembler.bx(target, ARMv7Assembler::JumpFixed); + m_assembler.bx(target); } // Address is a memory location containing the address to jump to void jump(Address address) { load32(address, dataTempRegister); - m_assembler.bx(dataTempRegister, ARMv7Assembler::JumpFixed); + m_assembler.bx(dataTempRegister); + } + + void jump(AbsoluteAddress address) + { + move(TrustedImmPtr(address.m_ptr), dataTempRegister); + load32(Address(dataTempRegister), dataTempRegister); + m_assembler.bx(dataTempRegister); } @@ -1028,107 +1454,173 @@ public: // * jo operations branch if the (signed) arithmetic // operation caused an overflow to occur. - Jump branchAdd32(Condition cond, RegisterID src, RegisterID dest) + Jump branchAdd32(ResultCondition cond, RegisterID op1, RegisterID op2, RegisterID dest) { - ASSERT((cond == Overflow) || (cond == Signed) || (cond == Zero) || (cond == NonZero)); - m_assembler.add_S(dest, dest, src); + m_assembler.add_S(dest, op1, op2); return Jump(makeBranch(cond)); } - Jump branchAdd32(Condition cond, Imm32 imm, RegisterID dest) + Jump branchAdd32(ResultCondition cond, RegisterID op1, TrustedImm32 imm, RegisterID dest) { - ASSERT((cond == Overflow) || (cond == Signed) || (cond == Zero) || (cond == NonZero)); ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); if (armImm.isValid()) - m_assembler.add_S(dest, dest, armImm); + m_assembler.add_S(dest, op1, armImm); else { move(imm, dataTempRegister); - m_assembler.add_S(dest, dest, dataTempRegister); + m_assembler.add_S(dest, op1, dataTempRegister); } return Jump(makeBranch(cond)); } - Jump branchMul32(Condition cond, RegisterID src, RegisterID dest) + Jump branchAdd32(ResultCondition cond, RegisterID src, RegisterID dest) { - ASSERT_UNUSED(cond, cond == Overflow); - m_assembler.smull(dest, dataTempRegister, dest, src); - m_assembler.asr(addressTempRegister, dest, 31); - return branch32(NotEqual, addressTempRegister, dataTempRegister); + return branchAdd32(cond, dest, src, dest); } - Jump branchMul32(Condition cond, Imm32 imm, RegisterID src, RegisterID dest) + Jump branchAdd32(ResultCondition cond, TrustedImm32 imm, RegisterID dest) + { + return branchAdd32(cond, dest, imm, dest); + } + + Jump branchAdd32(ResultCondition cond, TrustedImm32 imm, AbsoluteAddress dest) + { + // Move the high bits of the address into addressTempRegister, + // and load the value into dataTempRegister. + move(TrustedImmPtr(dest.m_ptr), addressTempRegister); + m_assembler.ldr(dataTempRegister, addressTempRegister, ARMThumbImmediate::makeUInt16(0)); + + // Do the add. + ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); + if (armImm.isValid()) + m_assembler.add_S(dataTempRegister, dataTempRegister, armImm); + else { + // If the operand does not fit into an immediate then load it temporarily + // into addressTempRegister; since we're overwriting addressTempRegister + // we'll need to reload it with the high bits of the address afterwards. + move(imm, addressTempRegister); + m_assembler.add_S(dataTempRegister, dataTempRegister, addressTempRegister); + move(TrustedImmPtr(dest.m_ptr), addressTempRegister); + } + + // Store the result. + m_assembler.str(dataTempRegister, addressTempRegister, ARMThumbImmediate::makeUInt16(0)); + + return Jump(makeBranch(cond)); + } + + Jump branchMul32(ResultCondition cond, RegisterID src1, RegisterID src2, RegisterID dest) + { + m_assembler.smull(dest, dataTempRegister, src1, src2); + + if (cond == Overflow) { + m_assembler.asr(addressTempRegister, dest, 31); + return branch32(NotEqual, addressTempRegister, dataTempRegister); + } + + return branchTest32(cond, dest); + } + + Jump branchMul32(ResultCondition cond, RegisterID src, RegisterID dest) + { + return branchMul32(cond, src, dest, dest); + } + + Jump branchMul32(ResultCondition cond, TrustedImm32 imm, RegisterID src, RegisterID dest) { - ASSERT_UNUSED(cond, cond == Overflow); move(imm, dataTempRegister); - m_assembler.smull(dest, dataTempRegister, src, dataTempRegister); - m_assembler.asr(addressTempRegister, dest, 31); - return branch32(NotEqual, addressTempRegister, dataTempRegister); + return branchMul32(cond, dataTempRegister, src, dest); + } + + Jump branchNeg32(ResultCondition cond, RegisterID srcDest) + { + ARMThumbImmediate zero = ARMThumbImmediate::makeUInt12(0); + m_assembler.sub_S(srcDest, zero, srcDest); + return Jump(makeBranch(cond)); } - Jump branchOr32(Condition cond, RegisterID src, RegisterID dest) + Jump branchOr32(ResultCondition cond, RegisterID src, RegisterID dest) { - ASSERT((cond == Signed) || (cond == Zero) || (cond == NonZero)); m_assembler.orr_S(dest, dest, src); return Jump(makeBranch(cond)); } - Jump branchSub32(Condition cond, RegisterID src, RegisterID dest) + Jump branchSub32(ResultCondition cond, RegisterID op1, RegisterID op2, RegisterID dest) { - ASSERT((cond == Overflow) || (cond == Signed) || (cond == Zero) || (cond == NonZero)); - m_assembler.sub_S(dest, dest, src); + m_assembler.sub_S(dest, op1, op2); return Jump(makeBranch(cond)); } - Jump branchSub32(Condition cond, Imm32 imm, RegisterID dest) + Jump branchSub32(ResultCondition cond, RegisterID op1, TrustedImm32 imm, RegisterID dest) { - ASSERT((cond == Overflow) || (cond == Signed) || (cond == Zero) || (cond == NonZero)); ARMThumbImmediate armImm = ARMThumbImmediate::makeEncodedImm(imm.m_value); if (armImm.isValid()) - m_assembler.sub_S(dest, dest, armImm); + m_assembler.sub_S(dest, op1, armImm); else { move(imm, dataTempRegister); - m_assembler.sub_S(dest, dest, dataTempRegister); + m_assembler.sub_S(dest, op1, dataTempRegister); } return Jump(makeBranch(cond)); } + Jump branchSub32(ResultCondition cond, RegisterID src, RegisterID dest) + { + return branchSub32(cond, dest, src, dest); + } + + Jump branchSub32(ResultCondition cond, TrustedImm32 imm, RegisterID dest) + { + return branchSub32(cond, dest, imm, dest); + } + + void relativeTableJump(RegisterID index, int scale) + { + ASSERT(scale >= 0 && scale <= 31); + + // dataTempRegister will point after the jump if index register contains zero + move(ARMRegisters::pc, dataTempRegister); + m_assembler.add(dataTempRegister, dataTempRegister, ARMThumbImmediate::makeEncodedImm(9)); + + ShiftTypeAndAmount shift(SRType_LSL, scale); + m_assembler.add(dataTempRegister, dataTempRegister, index, shift); + jump(dataTempRegister); + } // Miscellaneous operations: - void breakpoint() + void breakpoint(uint8_t imm = 0) { - m_assembler.bkpt(0); + m_assembler.bkpt(imm); } - Call nearCall() + ALWAYS_INLINE Call nearCall() { - moveFixedWidthEncoding(Imm32(0), dataTempRegister); - return Call(m_assembler.blx(dataTempRegister, ARMv7Assembler::JumpFixed), Call::LinkableNear); + moveFixedWidthEncoding(TrustedImm32(0), dataTempRegister); + return Call(m_assembler.blx(dataTempRegister), Call::LinkableNear); } - Call call() + ALWAYS_INLINE Call call() { - moveFixedWidthEncoding(Imm32(0), dataTempRegister); - return Call(m_assembler.blx(dataTempRegister, ARMv7Assembler::JumpFixed), Call::Linkable); + moveFixedWidthEncoding(TrustedImm32(0), dataTempRegister); + return Call(m_assembler.blx(dataTempRegister), Call::Linkable); } - Call call(RegisterID target) + ALWAYS_INLINE Call call(RegisterID target) { - return Call(m_assembler.blx(target, ARMv7Assembler::JumpFixed), Call::None); + return Call(m_assembler.blx(target), Call::None); } - Call call(Address address) + ALWAYS_INLINE Call call(Address address) { load32(address, dataTempRegister); - return Call(m_assembler.blx(dataTempRegister, ARMv7Assembler::JumpFixed), Call::None); + return Call(m_assembler.blx(dataTempRegister), Call::None); } - void ret() + ALWAYS_INLINE void ret() { - m_assembler.bx(linkRegister, ARMv7Assembler::JumpFixed); + m_assembler.bx(linkRegister); } - void set32(Condition cond, RegisterID left, RegisterID right, RegisterID dest) + void compare32(RelationalCondition cond, RegisterID left, RegisterID right, RegisterID dest) { m_assembler.cmp(left, right); m_assembler.it(armV7Condition(cond), false); @@ -1136,13 +1628,19 @@ public: m_assembler.mov(dest, ARMThumbImmediate::makeUInt16(0)); } - void set32(Condition cond, Address left, RegisterID right, RegisterID dest) + void compare32(RelationalCondition cond, Address left, RegisterID right, RegisterID dest) { load32(left, dataTempRegister); - set32(cond, dataTempRegister, right, dest); + compare32(cond, dataTempRegister, right, dest); } - void set32(Condition cond, RegisterID left, Imm32 right, RegisterID dest) + void compare8(RelationalCondition cond, Address left, TrustedImm32 right, RegisterID dest) + { + load8(left, addressTempRegister); + compare32(cond, addressTempRegister, right, dest); + } + + void compare32(RelationalCondition cond, RegisterID left, TrustedImm32 right, RegisterID dest) { compare32(left, right); m_assembler.it(armV7Condition(cond), false); @@ -1150,26 +1648,11 @@ public: m_assembler.mov(dest, ARMThumbImmediate::makeUInt16(0)); } - void set8(Condition cond, RegisterID left, RegisterID right, RegisterID dest) - { - set32(cond, left, right, dest); - } - - void set8(Condition cond, Address left, RegisterID right, RegisterID dest) - { - set32(cond, left, right, dest); - } - - void set8(Condition cond, RegisterID left, Imm32 right, RegisterID dest) - { - set32(cond, left, right, dest); - } - // FIXME: // The mask should be optional... paerhaps the argument order should be // dest-src, operations always have a dest? ... possibly not true, considering // asm ops like test, or pseudo ops like pop(). - void setTest32(Condition cond, Address address, Imm32 mask, RegisterID dest) + void test32(ResultCondition cond, Address address, TrustedImm32 mask, RegisterID dest) { load32(address, dataTempRegister); test32(dataTempRegister, mask); @@ -1178,7 +1661,7 @@ public: m_assembler.mov(dest, ARMThumbImmediate::makeUInt16(0)); } - void setTest8(Condition cond, Address address, Imm32 mask, RegisterID dest) + void test8(ResultCondition cond, Address address, TrustedImm32 mask, RegisterID dest) { load8(address, dataTempRegister); test32(dataTempRegister, mask); @@ -1187,48 +1670,91 @@ public: m_assembler.mov(dest, ARMThumbImmediate::makeUInt16(0)); } - DataLabel32 moveWithPatch(Imm32 imm, RegisterID dst) + ALWAYS_INLINE DataLabel32 moveWithPatch(TrustedImm32 imm, RegisterID dst) { + padBeforePatch(); moveFixedWidthEncoding(imm, dst); return DataLabel32(this); } - DataLabelPtr moveWithPatch(ImmPtr imm, RegisterID dst) + ALWAYS_INLINE DataLabelPtr moveWithPatch(TrustedImmPtr imm, RegisterID dst) { - moveFixedWidthEncoding(Imm32(imm), dst); + padBeforePatch(); + moveFixedWidthEncoding(TrustedImm32(imm), dst); return DataLabelPtr(this); } - Jump branchPtrWithPatch(Condition cond, RegisterID left, DataLabelPtr& dataLabel, ImmPtr initialRightValue = ImmPtr(0)) + ALWAYS_INLINE Jump branchPtrWithPatch(RelationalCondition cond, RegisterID left, DataLabelPtr& dataLabel, TrustedImmPtr initialRightValue = TrustedImmPtr(0)) { dataLabel = moveWithPatch(initialRightValue, dataTempRegister); return branch32(cond, left, dataTempRegister); } - Jump branchPtrWithPatch(Condition cond, Address left, DataLabelPtr& dataLabel, ImmPtr initialRightValue = ImmPtr(0)) + ALWAYS_INLINE Jump branchPtrWithPatch(RelationalCondition cond, Address left, DataLabelPtr& dataLabel, TrustedImmPtr initialRightValue = TrustedImmPtr(0)) { load32(left, addressTempRegister); dataLabel = moveWithPatch(initialRightValue, dataTempRegister); return branch32(cond, addressTempRegister, dataTempRegister); } + + PatchableJump patchableBranchPtr(RelationalCondition cond, Address left, TrustedImmPtr right = TrustedImmPtr(0)) + { + m_makeJumpPatchable = true; + Jump result = branch32(cond, left, TrustedImm32(right)); + m_makeJumpPatchable = false; + return PatchableJump(result); + } + + PatchableJump patchableBranchTest32(ResultCondition cond, RegisterID reg, TrustedImm32 mask = TrustedImm32(-1)) + { + m_makeJumpPatchable = true; + Jump result = branchTest32(cond, reg, mask); + m_makeJumpPatchable = false; + return PatchableJump(result); + } + + PatchableJump patchableBranch32(RelationalCondition cond, RegisterID reg, TrustedImm32 imm) + { + m_makeJumpPatchable = true; + Jump result = branch32(cond, reg, imm); + m_makeJumpPatchable = false; + return PatchableJump(result); + } + + PatchableJump patchableBranchPtrWithPatch(RelationalCondition cond, Address left, DataLabelPtr& dataLabel, TrustedImmPtr initialRightValue = TrustedImmPtr(0)) + { + m_makeJumpPatchable = true; + Jump result = branchPtrWithPatch(cond, left, dataLabel, initialRightValue); + m_makeJumpPatchable = false; + return PatchableJump(result); + } + + PatchableJump patchableJump() + { + padBeforePatch(); + m_makeJumpPatchable = true; + Jump result = jump(); + m_makeJumpPatchable = false; + return PatchableJump(result); + } - DataLabelPtr storePtrWithPatch(ImmPtr initialValue, ImplicitAddress address) + ALWAYS_INLINE DataLabelPtr storePtrWithPatch(TrustedImmPtr initialValue, ImplicitAddress address) { DataLabelPtr label = moveWithPatch(initialValue, dataTempRegister); store32(dataTempRegister, address); return label; } - DataLabelPtr storePtrWithPatch(ImplicitAddress address) { return storePtrWithPatch(ImmPtr(0), address); } + ALWAYS_INLINE DataLabelPtr storePtrWithPatch(ImplicitAddress address) { return storePtrWithPatch(TrustedImmPtr(0), address); } - Call tailRecursiveCall() + ALWAYS_INLINE Call tailRecursiveCall() { // Like a normal call, but don't link. - moveFixedWidthEncoding(Imm32(0), dataTempRegister); - return Call(m_assembler.bx(dataTempRegister, ARMv7Assembler::JumpFixed), Call::Linkable); + moveFixedWidthEncoding(TrustedImm32(0), dataTempRegister); + return Call(m_assembler.bx(dataTempRegister), Call::Linkable); } - Call makeTailRecursiveCall(Jump oldJump) + ALWAYS_INLINE Call makeTailRecursiveCall(Jump oldJump) { oldJump.link(this); return tailRecursiveCall(); @@ -1240,26 +1766,58 @@ public: return m_assembler.executableOffsetFor(location); } -protected: - bool inUninterruptedSequence() + static FunctionPtr readCallTarget(CodeLocationCall call) + { + return FunctionPtr(reinterpret_cast(ARMv7Assembler::readCallTarget(call.dataLocation()))); + } + + static bool canJumpReplacePatchableBranchPtrWithPatch() { return false; } + + static CodeLocationLabel startOfBranchPtrWithPatchOnRegister(CodeLocationDataLabelPtr label) + { + const unsigned twoWordOpSize = 4; + return label.labelAtOffset(-twoWordOpSize * 2); + } + + static void revertJumpReplacementToBranchPtrWithPatch(CodeLocationLabel instructionStart, RegisterID rd, void* initialValue) + { +#if OS(LINUX) || OS(QNX) + ARMv7Assembler::revertJumpTo_movT3movtcmpT2(instructionStart.dataLocation(), rd, dataTempRegister, reinterpret_cast(initialValue)); +#else + UNUSED_PARAM(rd); + ARMv7Assembler::revertJumpTo_movT3(instructionStart.dataLocation(), dataTempRegister, ARMThumbImmediate::makeUInt16(reinterpret_cast(initialValue) & 0xffff)); +#endif + } + + static CodeLocationLabel startOfPatchableBranchPtrWithPatchOnAddress(CodeLocationDataLabelPtr) { - return m_inUninterruptedSequence; + UNREACHABLE_FOR_PLATFORM(); + return CodeLocationLabel(); + } + + static void revertJumpReplacementToPatchableBranchPtrWithPatch(CodeLocationLabel, Address, void*) + { + UNREACHABLE_FOR_PLATFORM(); } - ARMv7Assembler::JmpSrc makeJump() +protected: + ALWAYS_INLINE Jump jump() { - moveFixedWidthEncoding(Imm32(0), dataTempRegister); - return m_assembler.bx(dataTempRegister, inUninterruptedSequence() ? ARMv7Assembler::JumpNoConditionFixedSize : ARMv7Assembler::JumpNoCondition); + m_assembler.label(); // Force nop-padding if we're in the middle of a watchpoint. + moveFixedWidthEncoding(TrustedImm32(0), dataTempRegister); + return Jump(m_assembler.bx(dataTempRegister), m_makeJumpPatchable ? ARMv7Assembler::JumpNoConditionFixedSize : ARMv7Assembler::JumpNoCondition); } - ARMv7Assembler::JmpSrc makeBranch(ARMv7Assembler::Condition cond) + ALWAYS_INLINE Jump makeBranch(ARMv7Assembler::Condition cond) { + m_assembler.label(); // Force nop-padding if we're in the middle of a watchpoint. m_assembler.it(cond, true, true); - moveFixedWidthEncoding(Imm32(0), dataTempRegister); - return m_assembler.bx(dataTempRegister, inUninterruptedSequence() ? ARMv7Assembler::JumpConditionFixedSize : ARMv7Assembler::JumpCondition, cond); + moveFixedWidthEncoding(TrustedImm32(0), dataTempRegister); + return Jump(m_assembler.bx(dataTempRegister), m_makeJumpPatchable ? ARMv7Assembler::JumpConditionFixedSize : ARMv7Assembler::JumpCondition, cond); } - ARMv7Assembler::JmpSrc makeBranch(Condition cond) { return makeBranch(armV7Condition(cond)); } - ARMv7Assembler::JmpSrc makeBranch(DoubleCondition cond) { return makeBranch(armV7Condition(cond)); } + ALWAYS_INLINE Jump makeBranch(RelationalCondition cond) { return makeBranch(armV7Condition(cond)); } + ALWAYS_INLINE Jump makeBranch(ResultCondition cond) { return makeBranch(armV7Condition(cond)); } + ALWAYS_INLINE Jump makeBranch(DoubleCondition cond) { return makeBranch(armV7Condition(cond)); } ArmAddress setupArmAddress(BaseIndex address) { @@ -1268,7 +1826,7 @@ protected: if (imm.isValid()) m_assembler.add(addressTempRegister, address.base, imm); else { - move(Imm32(address.offset), addressTempRegister); + move(TrustedImm32(address.offset), addressTempRegister); m_assembler.add(addressTempRegister, addressTempRegister, address.base); } @@ -1282,7 +1840,7 @@ protected: if ((address.offset >= -0xff) && (address.offset <= 0xfff)) return ArmAddress(address.base, address.offset); - move(Imm32(address.offset), addressTempRegister); + move(TrustedImm32(address.offset), addressTempRegister); return ArmAddress(address.base, addressTempRegister); } @@ -1291,7 +1849,7 @@ protected: if ((address.offset >= -0xff) && (address.offset <= 0xfff)) return ArmAddress(address.base, address.offset); - move(Imm32(address.offset), addressTempRegister); + move(TrustedImm32(address.offset), addressTempRegister); return ArmAddress(address.base, addressTempRegister); } @@ -1304,37 +1862,42 @@ protected: if (imm.isValid()) m_assembler.add(addressTempRegister, address.base, imm); else { - move(Imm32(address.offset), addressTempRegister); + move(TrustedImm32(address.offset), addressTempRegister); m_assembler.add(addressTempRegister, addressTempRegister, address.base); } return addressTempRegister; } - void moveFixedWidthEncoding(Imm32 imm, RegisterID dst) + void moveFixedWidthEncoding(TrustedImm32 imm, RegisterID dst) { uint32_t value = imm.m_value; m_assembler.movT3(dst, ARMThumbImmediate::makeUInt16(value & 0xffff)); m_assembler.movt(dst, ARMThumbImmediate::makeUInt16(value >> 16)); } - ARMv7Assembler::Condition armV7Condition(Condition cond) + ARMv7Assembler::Condition armV7Condition(RelationalCondition cond) { return static_cast(cond); } - ARMv7Assembler::Condition armV7Condition(DoubleCondition cond) + ARMv7Assembler::Condition armV7Condition(ResultCondition cond) { return static_cast(cond); } + ARMv7Assembler::Condition armV7Condition(DoubleCondition cond) + { + return static_cast(cond); + } + private: friend class LinkBuffer; friend class RepatchBuffer; static void linkCall(void* code, Call call, FunctionPtr function) { - ARMv7Assembler::linkCall(code, call.m_jmp, function.value()); + ARMv7Assembler::linkCall(code, call.m_label, function.value()); } static void repatchCall(CodeLocationCall call, CodeLocationLabel destination) @@ -1346,8 +1909,8 @@ private: { ARMv7Assembler::relinkCall(call.dataLocation(), destination.executableAddress()); } - - bool m_inUninterruptedSequence; + + bool m_makeJumpPatchable; }; } // namespace JSC