assembler/ARM64Assembler.h

   1 /*
   2  * Copyright (C) 2012, 2014 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #ifndef ARM64Assembler_h
  27 #define ARM64Assembler_h
  28
  29 #if ENABLE(ASSEMBLER) && CPU(ARM64)
  30
  31 #include "AssemblerBuffer.h"
  32 #include <limits.h>
  33 #include <wtf/Assertions.h>
  34 #include <wtf/Vector.h>
  35 #include <stdint.h>
  36
  37 #define CHECK_DATASIZE_OF(datasize) ASSERT(datasize == 32 || datasize == 64)
  38 #define DATASIZE_OF(datasize) ((datasize == 64) ? Datasize_64 : Datasize_32)
  39 #define MEMOPSIZE_OF(datasize) ((datasize == 8 || datasize == 128) ? MemOpSize_8_or_128 : (datasize == 16) ? MemOpSize_16 : (datasize == 32) ? MemOpSize_32 : MemOpSize_64)
  40 #define CHECK_DATASIZE() CHECK_DATASIZE_OF(datasize)
  41 #define DATASIZE DATASIZE_OF(datasize)
  42 #define MEMOPSIZE MEMOPSIZE_OF(datasize)
  43 #define CHECK_FP_MEMOP_DATASIZE() ASSERT(datasize == 8 || datasize == 16 || datasize == 32 || datasize == 64 || datasize == 128)
  44 #define MEMPAIROPSIZE_INT(datasize) ((datasize == 64) ? MemPairOp_64 : MemPairOp_32)
  45 #define MEMPAIROPSIZE_FP(datasize) ((datasize == 128) ? MemPairOp_V128 : (datasize == 64) ? MemPairOp_V64 : MemPairOp_32)
  46
  47 namespace JSC {
  48
  49 ALWAYS_INLINE bool isInt7(int32_t value)
  50 {
  51     return value == ((value << 25) >> 25);
  52 }
  53
  54 ALWAYS_INLINE bool isInt9(int32_t value)
  55 {
  56     return value == ((value << 23) >> 23);
  57 }
  58
  59 ALWAYS_INLINE bool isInt11(int32_t value)
  60 {
  61     return value == ((value << 21) >> 21);
  62 }
  63
  64 ALWAYS_INLINE bool isUInt5(int32_t value)
  65 {
  66     return !(value & ~0x1f);
  67 }
  68
  69 ALWAYS_INLINE bool isUInt12(int32_t value)
  70 {
  71     return !(value & ~0xfff);
  72 }
  73
  74 ALWAYS_INLINE bool isUInt12(intptr_t value)
  75 {
  76     return !(value & ~0xfffL);
  77 }
  78
  79 class UInt5 {
  80 public:
  81     explicit UInt5(int value)
  82         : m_value(value)
  83     {
  84         ASSERT(isUInt5(value));
  85     }
  86
  87     operator int() { return m_value; }
  88
  89 private:
  90     int m_value;
  91 };
  92
  93 class UInt12 {
  94 public:
  95     explicit UInt12(int value)
  96         : m_value(value)
  97     {
  98         ASSERT(isUInt12(value));
  99     }
 100
 101     operator int() { return m_value; }
 102
 103 private:
 104     int m_value;
 105 };
 106
 107 class PostIndex {
 108 public:
 109     explicit PostIndex(int value)
 110         : m_value(value)
 111     {
 112         ASSERT(isInt9(value));
 113     }
 114
 115     operator int() { return m_value; }
 116
 117 private:
 118     int m_value;
 119 };
 120
 121 class PreIndex {
 122 public:
 123     explicit PreIndex(int value)
 124         : m_value(value)
 125     {
 126         ASSERT(isInt9(value));
 127     }
 128
 129     operator int() { return m_value; }
 130
 131 private:
 132     int m_value;
 133 };
 134
 135 class PairPostIndex {
 136 public:
 137     explicit PairPostIndex(int value)
 138         : m_value(value)
 139     {
 140         ASSERT(isInt11(value));
 141     }
 142
 143     operator int() { return m_value; }
 144
 145 private:
 146     int m_value;
 147 };
 148
 149 class PairPreIndex {
 150 public:
 151     explicit PairPreIndex(int value)
 152         : m_value(value)
 153     {
 154         ASSERT(isInt11(value));
 155     }
 156
 157     operator int() { return m_value; }
 158
 159 private:
 160     int m_value;
 161 };
 162
 163 class LogicalImmediate {
 164 public:
 165     static LogicalImmediate create32(uint32_t value)
 166     {
 167         // Check for 0, -1 - these cannot be encoded.
 168         if (!value || !~value)
 169             return InvalidLogicalImmediate;
 170
 171         // First look for a 32-bit pattern, then for repeating 16-bit
 172         // patterns, 8-bit, 4-bit, and finally 2-bit.
 173
 174         unsigned hsb, lsb;
 175         bool inverted;
 176         if (findBitRange<32>(value, hsb, lsb, inverted))
 177             return encodeLogicalImmediate<32>(hsb, lsb, inverted);
 178
 179         if ((value & 0xffff) != (value >> 16))
 180             return InvalidLogicalImmediate;
 181         value &= 0xffff;
 182
 183         if (findBitRange<16>(value, hsb, lsb, inverted))
 184             return encodeLogicalImmediate<16>(hsb, lsb, inverted);
 185
 186         if ((value & 0xff) != (value >> 8))
 187             return InvalidLogicalImmediate;
 188         value &= 0xff;
 189
 190         if (findBitRange<8>(value, hsb, lsb, inverted))
 191             return encodeLogicalImmediate<8>(hsb, lsb, inverted);
 192
 193         if ((value & 0xf) != (value >> 4))
 194             return InvalidLogicalImmediate;
 195         value &= 0xf;
 196
 197         if (findBitRange<4>(value, hsb, lsb, inverted))
 198             return encodeLogicalImmediate<4>(hsb, lsb, inverted);
 199
 200         if ((value & 0x3) != (value >> 2))
 201             return InvalidLogicalImmediate;
 202         value &= 0x3;
 203
 204         if (findBitRange<2>(value, hsb, lsb, inverted))
 205             return encodeLogicalImmediate<2>(hsb, lsb, inverted);
 206
 207         return InvalidLogicalImmediate;
 208     }
 209
 210     static LogicalImmediate create64(uint64_t value)
 211     {
 212         // Check for 0, -1 - these cannot be encoded.
 213         if (!value || !~value)
 214             return InvalidLogicalImmediate;
 215
 216         // Look for a contiguous bit range.
 217         unsigned hsb, lsb;
 218         bool inverted;
 219         if (findBitRange<64>(value, hsb, lsb, inverted))
 220             return encodeLogicalImmediate<64>(hsb, lsb, inverted);
 221
 222         // If the high & low 32 bits are equal, we can try for a 32-bit (or narrower) pattern.
 223         if (static_cast<uint32_t>(value) == static_cast<uint32_t>(value >> 32))
 224             return create32(static_cast<uint32_t>(value));
 225         return InvalidLogicalImmediate;
 226     }
 227
 228     int value() const
 229     {
 230         ASSERT(isValid());
 231         return m_value;
 232     }
 233
 234     bool isValid() const
 235     {
 236         return m_value != InvalidLogicalImmediate;
 237     }
 238
 239     bool is64bit() const
 240     {
 241         return m_value & (1 << 12);
 242     }
 243
 244 private:
 245     LogicalImmediate(int value)
 246         : m_value(value)
 247     {
 248     }
 249
 250     // Generate a mask with bits in the range hsb..0 set, for example:
 251     //   hsb:63 = 0xffffffffffffffff
 252     //   hsb:42 = 0x000007ffffffffff
 253     //   hsb: 0 = 0x0000000000000001
 254     static uint64_t mask(unsigned hsb)
 255     {
 256         ASSERT(hsb < 64);
 257         return 0xffffffffffffffffull >> (63 - hsb);
 258     }
 259
 260     template<unsigned N>
 261     static void partialHSB(uint64_t& value, unsigned&result)
 262     {
 263         if (value & (0xffffffffffffffffull << N)) {
 264             result += N;
 265             value >>= N;
 266         }
 267     }
 268
 269     // Find the bit number of the highest bit set in a non-zero value, for example:
 270     //   0x8080808080808080 = hsb:63
 271     //   0x0000000000000001 = hsb: 0
 272     //   0x000007ffffe00000 = hsb:42
 273     static unsigned highestSetBit(uint64_t value)
 274     {
 275         ASSERT(value);
 276         unsigned hsb = 0;
 277         partialHSB<32>(value, hsb);
 278         partialHSB<16>(value, hsb);
 279         partialHSB<8>(value, hsb);
 280         partialHSB<4>(value, hsb);
 281         partialHSB<2>(value, hsb);
 282         partialHSB<1>(value, hsb);
 283         return hsb;
 284     }
 285
 286     // This function takes a value and a bit width, where value obeys the following constraints:
 287     //   * bits outside of the width of the value must be zero.
 288     //   * bits within the width of value must neither be all clear or all set.
 289     // The input is inspected to detect values that consist of either two or three contiguous
 290     // ranges of bits. The output range hsb..lsb will describe the second range of the value.
 291     // if the range is set, inverted will be false, and if the range is clear, inverted will
 292     // be true. For example (with width 8):
 293     //   00001111 = hsb:3, lsb:0, inverted:false
 294     //   11110000 = hsb:3, lsb:0, inverted:true
 295     //   00111100 = hsb:5, lsb:2, inverted:false
 296     //   11000011 = hsb:5, lsb:2, inverted:true
 297     template<unsigned width>
 298     static bool findBitRange(uint64_t value, unsigned& hsb, unsigned& lsb, bool& inverted)
 299     {
 300         ASSERT(value & mask(width - 1));
 301         ASSERT(value != mask(width - 1));
 302         ASSERT(!(value & ~mask(width - 1)));
 303
 304         // Detect cases where the top bit is set; if so, flip all the bits & set invert.
 305         // This halves the number of patterns we need to look for.
 306         const uint64_t msb = 1ull << (width - 1);
 307         if ((inverted = (value & msb)))
 308             value ^= mask(width - 1);
 309
 310         // Find the highest set bit in value, generate a corresponding mask & flip all
 311         // bits under it.
 312         hsb = highestSetBit(value);
 313         value ^= mask(hsb);
 314         if (!value) {
 315             // If this cleared the value, then the range hsb..0 was all set.
 316             lsb = 0;
 317             return true;
 318         }
 319
 320         // Try making one more mask, and flipping the bits!
 321         lsb = highestSetBit(value);
 322         value ^= mask(lsb);
 323         if (!value) {
 324             // Success - but lsb actually points to the hsb of a third range - add one
 325             // to get to the lsb of the mid range.
 326             ++lsb;
 327             return true;
 328         }
 329
 330         return false;
 331     }
 332
 333     // Encodes the set of immN:immr:imms fields found in a logical immediate.
 334     template<unsigned width>
 335     static int encodeLogicalImmediate(unsigned hsb, unsigned lsb, bool inverted)
 336     {
 337         // Check width is a power of 2!
 338         ASSERT(!(width & (width -1)));
 339         ASSERT(width <= 64 && width >= 2);
 340         ASSERT(hsb >= lsb);
 341         ASSERT(hsb < width);
 342
 343         int immN = 0;
 344         int imms = 0;
 345         int immr = 0;
 346
 347         // For 64-bit values this is easy - just set immN to true, and imms just
 348         // contains the bit number of the highest set bit of the set range. For
 349         // values with narrower widths, these are encoded by a leading set of
 350         // one bits, followed by a zero bit, followed by the remaining set of bits
 351         // being the high bit of the range. For a 32-bit immediate there are no
 352         // leading one bits, just a zero followed by a five bit number. For a
 353         // 16-bit immediate there is one one bit, a zero bit, and then a four bit
 354         // bit-position, etc.
 355         if (width == 64)
 356             immN = 1;
 357         else
 358             imms = 63 & ~(width + width - 1);
 359
 360         if (inverted) {
 361             // if width is 64 & hsb is 62, then we have a value something like:
 362             //   0x80000000ffffffff (in this case with lsb 32).
 363             // The ror should be by 1, imms (effectively set width minus 1) is
 364             // 32. Set width is full width minus cleared width.
 365             immr = (width - 1) - hsb;
 366             imms |= (width - ((hsb - lsb) + 1)) - 1;
 367         } else {
 368             // if width is 64 & hsb is 62, then we have a value something like:
 369             //   0x7fffffff00000000 (in this case with lsb 32).
 370             // The value is effectively rol'ed by lsb, which is equivalent to
 371             // a ror by width - lsb (or 0, in the case where lsb is 0). imms
 372             // is hsb - lsb.
 373             immr = (width - lsb) & (width - 1);
 374             imms |= hsb - lsb;
 375         }
 376
 377         return immN << 12 | immr << 6 | imms;
 378     }
 379
 380     static const int InvalidLogicalImmediate = -1;
 381
 382     int m_value;
 383 };
 384
 385 inline uint16_t getHalfword(uint64_t value, int which)
 386 {
 387     return value >> (which << 4);
 388 }
 389
 390 namespace ARM64Registers {
 391     typedef enum {
 392         // Parameter/result registers
 393         x0,
 394         x1,
 395         x2,
 396         x3,
 397         x4,
 398         x5,
 399         x6,
 400         x7,
 401         // Indirect result location register
 402         x8,
 403         // Temporary registers
 404         x9,
 405         x10,
 406         x11,
 407         x12,
 408         x13,
 409         x14,
 410         x15,
 411         // Intra-procedure-call scratch registers (temporary)
 412         x16, ip0 = x16,
 413         x17, ip1 = x17,
 414         // Platform Register (temporary)
 415         x18,
 416         // Callee-saved
 417         x19,
 418         x20,
 419         x21,
 420         x22,
 421         x23,
 422         x24,
 423         x25,
 424         x26,
 425         x27,
 426         x28,
 427         // Special
 428         x29, fp = x29,
 429         x30, lr = x30,
 430         sp,
 431         zr = 0x3f,
 432     } RegisterID;
 433
 434     typedef enum {
 435         // Parameter/result registers
 436         q0,
 437         q1,
 438         q2,
 439         q3,
 440         q4,
 441         q5,
 442         q6,
 443         q7,
 444         // Callee-saved (up to 64-bits only!)
 445         q8,
 446         q9,
 447         q10,
 448         q11,
 449         q12,
 450         q13,
 451         q14,
 452         q15,
 453         // Temporary registers
 454         q16,
 455         q17,
 456         q18,
 457         q19,
 458         q20,
 459         q21,
 460         q22,
 461         q23,
 462         q24,
 463         q25,
 464         q26,
 465         q27,
 466         q28,
 467         q29,
 468         q30,
 469         q31,
 470     } FPRegisterID;
 471
 472     static bool isSp(RegisterID reg) { return reg == sp; }
 473     static bool isZr(RegisterID reg) { return reg == zr; }
 474 }
 475
 476 class ARM64Assembler {
 477 public:
 478     typedef ARM64Registers::RegisterID RegisterID;
 479     typedef ARM64Registers::FPRegisterID FPRegisterID;
 480
 481     static RegisterID firstRegister() { return ARM64Registers::x0; }
 482     static RegisterID lastRegister() { return ARM64Registers::sp; }
 483
 484     static FPRegisterID firstFPRegister() { return ARM64Registers::q0; }
 485     static FPRegisterID lastFPRegister() { return ARM64Registers::q31; }
 486
 487 private:
 488     static bool isSp(RegisterID reg) { return ARM64Registers::isSp(reg); }
 489     static bool isZr(RegisterID reg) { return ARM64Registers::isZr(reg); }
 490
 491 public:
 492     ARM64Assembler()
 493         : m_indexOfLastWatchpoint(INT_MIN)
 494         , m_indexOfTailOfLastWatchpoint(INT_MIN)
 495     {
 496     }
 497
 498     AssemblerBuffer& buffer() { return m_buffer; }
 499
 500     // (HS, LO, HI, LS) -> (AE, B, A, BE)
 501     // (VS, VC) -> (O, NO)
 502     typedef enum {
 503         ConditionEQ,
 504         ConditionNE,
 505         ConditionHS, ConditionCS = ConditionHS,
 506         ConditionLO, ConditionCC = ConditionLO,
 507         ConditionMI,
 508         ConditionPL,
 509         ConditionVS,
 510         ConditionVC,
 511         ConditionHI,
 512         ConditionLS,
 513         ConditionGE,
 514         ConditionLT,
 515         ConditionGT,
 516         ConditionLE,
 517         ConditionAL,
 518         ConditionInvalid
 519     } Condition;
 520
 521     static Condition invert(Condition cond)
 522     {
 523         return static_cast<Condition>(cond ^ 1);
 524     }
 525
 526     typedef enum {
 527         LSL,
 528         LSR,
 529         ASR,
 530         ROR
 531     } ShiftType;
 532
 533     typedef enum {
 534         UXTB,
 535         UXTH,
 536         UXTW,
 537         UXTX,
 538         SXTB,
 539         SXTH,
 540         SXTW,
 541         SXTX
 542     } ExtendType;
 543
 544     enum SetFlags {
 545         DontSetFlags,
 546         S
 547     };
 548
 549 #define JUMP_ENUM_WITH_SIZE(index, value) (((value) << 4) | (index))
 550 #define JUMP_ENUM_SIZE(jump) ((jump) >> 4)
 551     enum JumpType { JumpFixed = JUMP_ENUM_WITH_SIZE(0, 0),
 552         JumpNoCondition = JUMP_ENUM_WITH_SIZE(1, 1 * sizeof(uint32_t)),
 553         JumpCondition = JUMP_ENUM_WITH_SIZE(2, 2 * sizeof(uint32_t)),
 554         JumpCompareAndBranch = JUMP_ENUM_WITH_SIZE(3, 2 * sizeof(uint32_t)),
 555         JumpTestBit = JUMP_ENUM_WITH_SIZE(4, 2 * sizeof(uint32_t)),
 556         JumpNoConditionFixedSize = JUMP_ENUM_WITH_SIZE(5, 1 * sizeof(uint32_t)),
 557         JumpConditionFixedSize = JUMP_ENUM_WITH_SIZE(6, 2 * sizeof(uint32_t)),
 558         JumpCompareAndBranchFixedSize = JUMP_ENUM_WITH_SIZE(7, 2 * sizeof(uint32_t)),
 559         JumpTestBitFixedSize = JUMP_ENUM_WITH_SIZE(8, 2 * sizeof(uint32_t)),
 560     };
 561     enum JumpLinkType {
 562         LinkInvalid = JUMP_ENUM_WITH_SIZE(0, 0),
 563         LinkJumpNoCondition = JUMP_ENUM_WITH_SIZE(1, 1 * sizeof(uint32_t)),
 564         LinkJumpConditionDirect = JUMP_ENUM_WITH_SIZE(2, 1 * sizeof(uint32_t)),
 565         LinkJumpCondition = JUMP_ENUM_WITH_SIZE(3, 2 * sizeof(uint32_t)),
 566         LinkJumpCompareAndBranch = JUMP_ENUM_WITH_SIZE(4, 2 * sizeof(uint32_t)),
 567         LinkJumpCompareAndBranchDirect = JUMP_ENUM_WITH_SIZE(5, 1 * sizeof(uint32_t)),
 568         LinkJumpTestBit = JUMP_ENUM_WITH_SIZE(6, 2 * sizeof(uint32_t)),
 569         LinkJumpTestBitDirect = JUMP_ENUM_WITH_SIZE(7, 1 * sizeof(uint32_t)),
 570     };
 571
 572     class LinkRecord {
 573     public:
 574         LinkRecord(intptr_t from, intptr_t to, JumpType type, Condition condition)
 575         {
 576             data.realTypes.m_from = from;
 577             data.realTypes.m_to = to;
 578             data.realTypes.m_type = type;
 579             data.realTypes.m_linkType = LinkInvalid;
 580             data.realTypes.m_condition = condition;
 581         }
 582         LinkRecord(intptr_t from, intptr_t to, JumpType type, Condition condition, bool is64Bit, RegisterID compareRegister)
 583         {
 584             data.realTypes.m_from = from;
 585             data.realTypes.m_to = to;
 586             data.realTypes.m_type = type;
 587             data.realTypes.m_linkType = LinkInvalid;
 588             data.realTypes.m_condition = condition;
 589             data.realTypes.m_is64Bit = is64Bit;
 590             data.realTypes.m_compareRegister = compareRegister;
 591         }
 592         LinkRecord(intptr_t from, intptr_t to, JumpType type, Condition condition, unsigned bitNumber, RegisterID compareRegister)
 593         {
 594             data.realTypes.m_from = from;
 595             data.realTypes.m_to = to;
 596             data.realTypes.m_type = type;
 597             data.realTypes.m_linkType = LinkInvalid;
 598             data.realTypes.m_condition = condition;
 599             data.realTypes.m_bitNumber = bitNumber;
 600             data.realTypes.m_compareRegister = compareRegister;
 601         }
 602         void operator=(const LinkRecord& other)
 603         {
 604             data.copyTypes.content[0] = other.data.copyTypes.content[0];
 605             data.copyTypes.content[1] = other.data.copyTypes.content[1];
 606             data.copyTypes.content[2] = other.data.copyTypes.content[2];
 607         }
 608         intptr_t from() const { return data.realTypes.m_from; }
 609         void setFrom(intptr_t from) { data.realTypes.m_from = from; }
 610         intptr_t to() const { return data.realTypes.m_to; }
 611         JumpType type() const { return data.realTypes.m_type; }
 612         JumpLinkType linkType() const { return data.realTypes.m_linkType; }
 613         void setLinkType(JumpLinkType linkType) { ASSERT(data.realTypes.m_linkType == LinkInvalid); data.realTypes.m_linkType = linkType; }
 614         Condition condition() const { return data.realTypes.m_condition; }
 615         bool is64Bit() const { return data.realTypes.m_is64Bit; }
 616         unsigned bitNumber() const { return data.realTypes.m_bitNumber; }
 617         RegisterID compareRegister() const { return data.realTypes.m_compareRegister; }
 618
 619     private:
 620         union {
 621             struct RealTypes {
 622                 intptr_t m_from : 48;
 623                 intptr_t m_to : 48;
 624                 JumpType m_type : 8;
 625                 JumpLinkType m_linkType : 8;
 626                 Condition m_condition : 4;
 627                 bool m_is64Bit : 1;
 628                 unsigned m_bitNumber : 6;
 629                 RegisterID m_compareRegister : 5;
 630             } realTypes;
 631             struct CopyTypes {
 632                 uint64_t content[3];
 633             } copyTypes;
 634             COMPILE_ASSERT(sizeof(RealTypes) == sizeof(CopyTypes), LinkRecordCopyStructSizeEqualsRealStruct);
 635         } data;
 636     };
 637
 638     // bits(N) VFPExpandImm(bits(8) imm8);
 639     //
 640     // Encoding of floating point immediates is a litte complicated. Here's a
 641     // high level description:
 642     //     +/-m*2-n where m and n are integers, 16 <= m <= 31, 0 <= n <= 7
 643     // and the algirithm for expanding to a single precision float:
 644     //     return imm8<7>:NOT(imm8<6>):Replicate(imm8<6>,5):imm8<5:0>:Zeros(19);
 645     //
 646     // The trickiest bit is how the exponent is handled. The following table
 647     // may help clarify things a little:
 648     //     654
 649     //     100 01111100 124 -3 1020 01111111100
 650     //     101 01111101 125 -2 1021 01111111101
 651     //     110 01111110 126 -1 1022 01111111110
 652     //     111 01111111 127  0 1023 01111111111
 653     //     000 10000000 128  1 1024 10000000000
 654     //     001 10000001 129  2 1025 10000000001
 655     //     010 10000010 130  3 1026 10000000010
 656     //     011 10000011 131  4 1027 10000000011
 657     // The first column shows the bit pattern stored in bits 6-4 of the arm
 658     // encoded immediate. The second column shows the 8-bit IEEE 754 single
 659     // -precision exponent in binary, the third column shows the raw decimal
 660     // value. IEEE 754 single-precision numbers are stored with a bias of 127
 661     // to the exponent, so the fourth column shows the resulting exponent.
 662     // From this was can see that the exponent can be in the range -3..4,
 663     // which agrees with the high level description given above. The fifth
 664     // and sixth columns shows the value stored in a IEEE 754 double-precision
 665     // number to represent these exponents in decimal and binary, given the
 666     // bias of 1023.
 667     //
 668     // Ultimately, detecting doubles that can be encoded as immediates on arm
 669     // and encoding doubles is actually not too bad. A floating point value can
 670     // be encoded by retaining the sign bit, the low three bits of the exponent
 671     // and the high 4 bits of the mantissa. To validly be able to encode an
 672     // immediate the remainder of the mantissa must be zero, and the high part
 673     // of the exponent must match the top bit retained, bar the highest bit
 674     // which must be its inverse.
 675     static bool canEncodeFPImm(double d)
 676     {
 677         // Discard the sign bit, the low two bits of the exponent & the highest
 678         // four bits of the mantissa.
 679         uint64_t masked = bitwise_cast<uint64_t>(d) & 0x7fc0ffffffffffffull;
 680         return (masked == 0x3fc0000000000000ull) || (masked == 0x4000000000000000ull);
 681     }
 682
 683     template<int datasize>
 684     static bool canEncodePImmOffset(int32_t offset)
 685     {
 686         int32_t maxPImm = 4095 * (datasize / 8);
 687         if (offset < 0)
 688             return false;
 689         if (offset > maxPImm)
 690             return false;
 691         if (offset & ((datasize / 8 ) - 1))
 692             return false;
 693         return true;
 694     }
 695
 696     static bool canEncodeSImmOffset(int32_t offset)
 697     {
 698         return isInt9(offset);
 699     }
 700
 701 private:
 702     int encodeFPImm(double d)
 703     {
 704         ASSERT(canEncodeFPImm(d));
 705         uint64_t u64 = bitwise_cast<uint64_t>(d);
 706         return (static_cast<int>(u64 >> 56) & 0x80) | (static_cast<int>(u64 >> 48) & 0x7f);
 707     }
 708
 709     template<int datasize>
 710     int encodeShiftAmount(int amount)
 711     {
 712         ASSERT(!amount || datasize == (8 << amount));
 713         return amount;
 714     }
 715
 716     template<int datasize>
 717     static int encodePositiveImmediate(unsigned pimm)
 718     {
 719         ASSERT(!(pimm & ((datasize / 8) - 1)));
 720         return pimm / (datasize / 8);
 721     }
 722
 723     enum Datasize {
 724         Datasize_32,
 725         Datasize_64,
 726         Datasize_64_top,
 727         Datasize_16
 728     };
 729
 730     enum MemOpSize {
 731         MemOpSize_8_or_128,
 732         MemOpSize_16,
 733         MemOpSize_32,
 734         MemOpSize_64,
 735     };
 736
 737     enum BranchType {
 738         BranchType_JMP,
 739         BranchType_CALL,
 740         BranchType_RET
 741     };
 742
 743     enum AddOp {
 744         AddOp_ADD,
 745         AddOp_SUB
 746     };
 747
 748     enum BitfieldOp {
 749         BitfieldOp_SBFM,
 750         BitfieldOp_BFM,
 751         BitfieldOp_UBFM
 752     };
 753
 754     enum DataOp1Source {
 755         DataOp_RBIT,
 756         DataOp_REV16,
 757         DataOp_REV32,
 758         DataOp_REV64,
 759         DataOp_CLZ,
 760         DataOp_CLS
 761     };
 762
 763     enum DataOp2Source {
 764         DataOp_UDIV = 2,
 765         DataOp_SDIV = 3,
 766         DataOp_LSLV = 8,
 767         DataOp_LSRV = 9,
 768         DataOp_ASRV = 10,
 769         DataOp_RORV = 11
 770     };
 771
 772     enum DataOp3Source {
 773         DataOp_MADD = 0,
 774         DataOp_MSUB = 1,
 775         DataOp_SMADDL = 2,
 776         DataOp_SMSUBL = 3,
 777         DataOp_SMULH = 4,
 778         DataOp_UMADDL = 10,
 779         DataOp_UMSUBL = 11,
 780         DataOp_UMULH = 12
 781     };
 782
 783     enum ExcepnOp {
 784         ExcepnOp_EXCEPTION = 0,
 785         ExcepnOp_BREAKPOINT = 1,
 786         ExcepnOp_HALT = 2,
 787         ExcepnOp_DCPS = 5
 788     };
 789
 790     enum FPCmpOp {
 791         FPCmpOp_FCMP = 0x00,
 792         FPCmpOp_FCMP0 = 0x08,
 793         FPCmpOp_FCMPE = 0x10,
 794         FPCmpOp_FCMPE0 = 0x18
 795     };
 796
 797     enum FPCondCmpOp {
 798         FPCondCmpOp_FCMP,
 799         FPCondCmpOp_FCMPE
 800     };
 801
 802     enum FPDataOp1Source {
 803         FPDataOp_FMOV = 0,
 804         FPDataOp_FABS = 1,
 805         FPDataOp_FNEG = 2,
 806         FPDataOp_FSQRT = 3,
 807         FPDataOp_FCVT_toSingle = 4,
 808         FPDataOp_FCVT_toDouble = 5,
 809         FPDataOp_FCVT_toHalf = 7,
 810         FPDataOp_FRINTN = 8,
 811         FPDataOp_FRINTP = 9,
 812         FPDataOp_FRINTM = 10,
 813         FPDataOp_FRINTZ = 11,
 814         FPDataOp_FRINTA = 12,
 815         FPDataOp_FRINTX = 14,
 816         FPDataOp_FRINTI = 15
 817     };
 818
 819     enum FPDataOp2Source {
 820         FPDataOp_FMUL,
 821         FPDataOp_FDIV,
 822         FPDataOp_FADD,
 823         FPDataOp_FSUB,
 824         FPDataOp_FMAX,
 825         FPDataOp_FMIN,
 826         FPDataOp_FMAXNM,
 827         FPDataOp_FMINNM,
 828         FPDataOp_FNMUL
 829     };
 830
 831     enum FPIntConvOp {
 832         FPIntConvOp_FCVTNS = 0x00,
 833         FPIntConvOp_FCVTNU = 0x01,
 834         FPIntConvOp_SCVTF = 0x02,
 835         FPIntConvOp_UCVTF = 0x03,
 836         FPIntConvOp_FCVTAS = 0x04,
 837         FPIntConvOp_FCVTAU = 0x05,
 838         FPIntConvOp_FMOV_QtoX = 0x06,
 839         FPIntConvOp_FMOV_XtoQ = 0x07,
 840         FPIntConvOp_FCVTPS = 0x08,
 841         FPIntConvOp_FCVTPU = 0x09,
 842         FPIntConvOp_FMOV_QtoX_top = 0x0e,
 843         FPIntConvOp_FMOV_XtoQ_top = 0x0f,
 844         FPIntConvOp_FCVTMS = 0x10,
 845         FPIntConvOp_FCVTMU = 0x11,
 846         FPIntConvOp_FCVTZS = 0x18,
 847         FPIntConvOp_FCVTZU = 0x19,
 848     };
 849
 850     enum LogicalOp {
 851         LogicalOp_AND,
 852         LogicalOp_ORR,
 853         LogicalOp_EOR,
 854         LogicalOp_ANDS
 855     };
 856
 857     enum MemOp {
 858         MemOp_STORE,
 859         MemOp_LOAD,
 860         MemOp_STORE_V128,
 861         MemOp_LOAD_V128,
 862         MemOp_PREFETCH = 2, // size must be 3
 863         MemOp_LOAD_signed64 = 2, // size may be 0, 1 or 2
 864         MemOp_LOAD_signed32 = 3 // size may be 0 or 1
 865     };
 866
 867     enum MemPairOpSize {
 868         MemPairOp_32 = 0,
 869         MemPairOp_LoadSigned_32 = 1,
 870         MemPairOp_64 = 2,
 871
 872         MemPairOp_V32 = MemPairOp_32,
 873         MemPairOp_V64 = 1,
 874         MemPairOp_V128 = 2
 875     };
 876
 877     enum MoveWideOp {
 878         MoveWideOp_N = 0,
 879         MoveWideOp_Z = 2,
 880         MoveWideOp_K = 3
 881     };
 882
 883     enum LdrLiteralOp {
 884         LdrLiteralOp_32BIT = 0,
 885         LdrLiteralOp_64BIT = 1,
 886         LdrLiteralOp_LDRSW = 2,
 887         LdrLiteralOp_128BIT = 2
 888     };
 889
 890     static unsigned memPairOffsetShift(bool V, MemPairOpSize size)
 891     {
 892         // return the log2 of the size in bytes, e.g. 64 bit size returns 3
 893         if (V)
 894             return size + 2;
 895         return (size >> 1) + 2;
 896     }
 897
 898 public:
 899     // Integer Instructions:
 900
 901     template<int datasize, SetFlags setFlags = DontSetFlags>
 902     ALWAYS_INLINE void adc(RegisterID rd, RegisterID rn, RegisterID rm)
 903     {
 904         CHECK_DATASIZE();
 905         insn(addSubtractWithCarry(DATASIZE, AddOp_ADD, setFlags, rm, rn, rd));
 906     }
 907
 908     template<int datasize, SetFlags setFlags = DontSetFlags>
 909     ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, UInt12 imm12, int shift = 0)
 910     {
 911         CHECK_DATASIZE();
 912         ASSERT(!shift || shift == 12);
 913         insn(addSubtractImmediate(DATASIZE, AddOp_ADD, setFlags, shift == 12, imm12, rn, rd));
 914     }
 915
 916     template<int datasize, SetFlags setFlags = DontSetFlags>
 917     ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, RegisterID rm)
 918     {
 919         add<datasize, setFlags>(rd, rn, rm, LSL, 0);
 920     }
 921
 922     template<int datasize, SetFlags setFlags = DontSetFlags>
 923     ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
 924     {
 925         CHECK_DATASIZE();
 926         insn(addSubtractExtendedRegister(DATASIZE, AddOp_ADD, setFlags, rm, extend, amount, rn, rd));
 927     }
 928
 929     template<int datasize, SetFlags setFlags = DontSetFlags>
 930     ALWAYS_INLINE void add(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)
 931     {
 932         CHECK_DATASIZE();
 933         if (isSp(rd) || isSp(rn)) {
 934             ASSERT(shift == LSL);
 935             ASSERT(!isSp(rm));
 936             add<datasize, setFlags>(rd, rn, rm, UXTX, amount);
 937         } else
 938             insn(addSubtractShiftedRegister(DATASIZE, AddOp_ADD, setFlags, shift, rm, amount, rn, rd));
 939     }
 940
 941     ALWAYS_INLINE void adr(RegisterID rd, int offset)
 942     {
 943         insn(pcRelative(false, offset, rd));
 944     }
 945
 946     ALWAYS_INLINE void adrp(RegisterID rd, int offset)
 947     {
 948         ASSERT(!(offset & 0xfff));
 949         insn(pcRelative(true, offset >> 12, rd));
 950     }
 951
 952     template<int datasize, SetFlags setFlags = DontSetFlags>
 953     ALWAYS_INLINE void and_(RegisterID rd, RegisterID rn, RegisterID rm)
 954     {
 955         and_<datasize, setFlags>(rd, rn, rm, LSL, 0);
 956     }
 957
 958     template<int datasize, SetFlags setFlags = DontSetFlags>
 959     ALWAYS_INLINE void and_(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)
 960     {
 961         CHECK_DATASIZE();
 962         insn(logicalShiftedRegister(DATASIZE, setFlags ? LogicalOp_ANDS : LogicalOp_AND, shift, false, rm, amount, rn, rd));
 963     }
 964
 965     template<int datasize, SetFlags setFlags = DontSetFlags>
 966     ALWAYS_INLINE void and_(RegisterID rd, RegisterID rn, LogicalImmediate imm)
 967     {
 968         CHECK_DATASIZE();
 969         insn(logicalImmediate(DATASIZE, setFlags ? LogicalOp_ANDS : LogicalOp_AND, imm.value(), rn, rd));
 970     }
 971
 972     template<int datasize>
 973     ALWAYS_INLINE void asr(RegisterID rd, RegisterID rn, int shift)
 974     {
 975         ASSERT(shift < datasize);
 976         sbfm<datasize>(rd, rn, shift, datasize - 1);
 977     }
 978
 979     template<int datasize>
 980     ALWAYS_INLINE void asr(RegisterID rd, RegisterID rn, RegisterID rm)
 981     {
 982         asrv<datasize>(rd, rn, rm);
 983     }
 984
 985     template<int datasize>
 986     ALWAYS_INLINE void asrv(RegisterID rd, RegisterID rn, RegisterID rm)
 987     {
 988         CHECK_DATASIZE();
 989         insn(dataProcessing2Source(DATASIZE, rm, DataOp_ASRV, rn, rd));
 990     }
 991
 992     ALWAYS_INLINE void b(int32_t offset = 0)
 993     {
 994         ASSERT(!(offset & 3));
 995         offset >>= 2;
 996         ASSERT(offset == (offset << 6) >> 6);
 997         insn(unconditionalBranchImmediate(false, offset));
 998     }
 999
1000     ALWAYS_INLINE void b_cond(Condition cond, int32_t offset = 0)
1001     {
1002         ASSERT(!(offset & 3));
1003         offset >>= 2;
1004         ASSERT(offset == (offset << 13) >> 13);
1005         insn(conditionalBranchImmediate(offset, cond));
1006     }
1007
1008     template<int datasize>
1009     ALWAYS_INLINE void bfi(RegisterID rd, RegisterID rn, int lsb, int width)
1010     {
1011         bfm<datasize>(rd, rn, (datasize - lsb) & (datasize - 1), width - 1);
1012     }
1013
1014     template<int datasize>
1015     ALWAYS_INLINE void bfm(RegisterID rd, RegisterID rn, int immr, int imms)
1016     {
1017         CHECK_DATASIZE();
1018         insn(bitfield(DATASIZE, BitfieldOp_BFM, immr, imms, rn, rd));
1019     }
1020
1021     template<int datasize>
1022     ALWAYS_INLINE void bfxil(RegisterID rd, RegisterID rn, int lsb, int width)
1023     {
1024         bfm<datasize>(rd, rn, lsb, lsb + width - 1);
1025     }
1026
1027     template<int datasize, SetFlags setFlags = DontSetFlags>
1028     ALWAYS_INLINE void bic(RegisterID rd, RegisterID rn, RegisterID rm)
1029     {
1030         bic<datasize, setFlags>(rd, rn, rm, LSL, 0);
1031     }
1032
1033     template<int datasize, SetFlags setFlags = DontSetFlags>
1034     ALWAYS_INLINE void bic(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)
1035     {
1036         CHECK_DATASIZE();
1037         insn(logicalShiftedRegister(DATASIZE, setFlags ? LogicalOp_ANDS : LogicalOp_AND, shift, true, rm, amount, rn, rd));
1038     }
1039
1040     ALWAYS_INLINE void bl(int32_t offset = 0)
1041     {
1042         ASSERT(!(offset & 3));
1043         offset >>= 2;
1044         insn(unconditionalBranchImmediate(true, offset));
1045     }
1046
1047     ALWAYS_INLINE void blr(RegisterID rn)
1048     {
1049         insn(unconditionalBranchRegister(BranchType_CALL, rn));
1050     }
1051
1052     ALWAYS_INLINE void br(RegisterID rn)
1053     {
1054         insn(unconditionalBranchRegister(BranchType_JMP, rn));
1055     }
1056
1057     ALWAYS_INLINE void brk(uint16_t imm)
1058     {
1059         insn(excepnGeneration(ExcepnOp_BREAKPOINT, imm, 0));
1060     }
1061
1062     template<int datasize>
1063     ALWAYS_INLINE void cbnz(RegisterID rt, int32_t offset = 0)
1064     {
1065         CHECK_DATASIZE();
1066         ASSERT(!(offset & 3));
1067         offset >>= 2;
1068         insn(compareAndBranchImmediate(DATASIZE, true, offset, rt));
1069     }
1070
1071     template<int datasize>
1072     ALWAYS_INLINE void cbz(RegisterID rt, int32_t offset = 0)
1073     {
1074         CHECK_DATASIZE();
1075         ASSERT(!(offset & 3));
1076         offset >>= 2;
1077         insn(compareAndBranchImmediate(DATASIZE, false, offset, rt));
1078     }
1079
1080     template<int datasize>
1081     ALWAYS_INLINE void ccmn(RegisterID rn, RegisterID rm, int nzcv, Condition cond)
1082     {
1083         CHECK_DATASIZE();
1084         insn(conditionalCompareRegister(DATASIZE, AddOp_ADD, rm, cond, rn, nzcv));
1085     }
1086
1087     template<int datasize>
1088     ALWAYS_INLINE void ccmn(RegisterID rn, UInt5 imm, int nzcv, Condition cond)
1089     {
1090         CHECK_DATASIZE();
1091         insn(conditionalCompareImmediate(DATASIZE, AddOp_ADD, imm, cond, rn, nzcv));
1092     }
1093
1094     template<int datasize>
1095     ALWAYS_INLINE void ccmp(RegisterID rn, RegisterID rm, int nzcv, Condition cond)
1096     {
1097         CHECK_DATASIZE();
1098         insn(conditionalCompareRegister(DATASIZE, AddOp_SUB, rm, cond, rn, nzcv));
1099     }
1100
1101     template<int datasize>
1102     ALWAYS_INLINE void ccmp(RegisterID rn, UInt5 imm, int nzcv, Condition cond)
1103     {
1104         CHECK_DATASIZE();
1105         insn(conditionalCompareImmediate(DATASIZE, AddOp_SUB, imm, cond, rn, nzcv));
1106     }
1107
1108     template<int datasize>
1109     ALWAYS_INLINE void cinc(RegisterID rd, RegisterID rn, Condition cond)
1110     {
1111         csinc<datasize>(rd, rn, rn, invert(cond));
1112     }
1113
1114     template<int datasize>
1115     ALWAYS_INLINE void cinv(RegisterID rd, RegisterID rn, Condition cond)
1116     {
1117         csinv<datasize>(rd, rn, rn, invert(cond));
1118     }
1119
1120     template<int datasize>
1121     ALWAYS_INLINE void cls(RegisterID rd, RegisterID rn)
1122     {
1123         CHECK_DATASIZE();
1124         insn(dataProcessing1Source(DATASIZE, DataOp_CLS, rn, rd));
1125     }
1126
1127     template<int datasize>
1128     ALWAYS_INLINE void clz(RegisterID rd, RegisterID rn)
1129     {
1130         CHECK_DATASIZE();
1131         insn(dataProcessing1Source(DATASIZE, DataOp_CLZ, rn, rd));
1132     }
1133
1134     template<int datasize>
1135     ALWAYS_INLINE void cmn(RegisterID rn, UInt12 imm12, int shift = 0)
1136     {
1137         add<datasize, S>(ARM64Registers::zr, rn, imm12, shift);
1138     }
1139
1140     template<int datasize>
1141     ALWAYS_INLINE void cmn(RegisterID rn, RegisterID rm)
1142     {
1143         add<datasize, S>(ARM64Registers::zr, rn, rm);
1144     }
1145
1146     template<int datasize>
1147     ALWAYS_INLINE void cmn(RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1148     {
1149         add<datasize, S>(ARM64Registers::zr, rn, rm, extend, amount);
1150     }
1151
1152     template<int datasize>
1153     ALWAYS_INLINE void cmn(RegisterID rn, RegisterID rm, ShiftType shift, int amount)
1154     {
1155         add<datasize, S>(ARM64Registers::zr, rn, rm, shift, amount);
1156     }
1157
1158     template<int datasize>
1159     ALWAYS_INLINE void cmp(RegisterID rn, UInt12 imm12, int shift = 0)
1160     {
1161         sub<datasize, S>(ARM64Registers::zr, rn, imm12, shift);
1162     }
1163
1164     template<int datasize>
1165     ALWAYS_INLINE void cmp(RegisterID rn, RegisterID rm)
1166     {
1167         sub<datasize, S>(ARM64Registers::zr, rn, rm);
1168     }
1169
1170     template<int datasize>
1171     ALWAYS_INLINE void cmp(RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1172     {
1173         sub<datasize, S>(ARM64Registers::zr, rn, rm, extend, amount);
1174     }
1175
1176     template<int datasize>
1177     ALWAYS_INLINE void cmp(RegisterID rn, RegisterID rm, ShiftType shift, int amount)
1178     {
1179         sub<datasize, S>(ARM64Registers::zr, rn, rm, shift, amount);
1180     }
1181
1182     template<int datasize>
1183     ALWAYS_INLINE void cneg(RegisterID rd, RegisterID rn, Condition cond)
1184     {
1185         csneg<datasize>(rd, rn, rn, invert(cond));
1186     }
1187
1188     template<int datasize>
1189     ALWAYS_INLINE void csel(RegisterID rd, RegisterID rn, RegisterID rm, Condition cond)
1190     {
1191         CHECK_DATASIZE();
1192         insn(conditionalSelect(DATASIZE, false, rm, cond, false, rn, rd));
1193     }
1194
1195     template<int datasize>
1196     ALWAYS_INLINE void cset(RegisterID rd, Condition cond)
1197     {
1198         csinc<datasize>(rd, ARM64Registers::zr, ARM64Registers::zr, invert(cond));
1199     }
1200
1201     template<int datasize>
1202     ALWAYS_INLINE void csetm(RegisterID rd, Condition cond)
1203     {
1204         csinv<datasize>(rd, ARM64Registers::zr, ARM64Registers::zr, invert(cond));
1205     }
1206
1207     template<int datasize>
1208     ALWAYS_INLINE void csinc(RegisterID rd, RegisterID rn, RegisterID rm, Condition cond)
1209     {
1210         CHECK_DATASIZE();
1211         insn(conditionalSelect(DATASIZE, false, rm, cond, true, rn, rd));
1212     }
1213
1214     template<int datasize>
1215     ALWAYS_INLINE void csinv(RegisterID rd, RegisterID rn, RegisterID rm, Condition cond)
1216     {
1217         CHECK_DATASIZE();
1218         insn(conditionalSelect(DATASIZE, true, rm, cond, false, rn, rd));
1219     }
1220
1221     template<int datasize>
1222     ALWAYS_INLINE void csneg(RegisterID rd, RegisterID rn, RegisterID rm, Condition cond)
1223     {
1224         CHECK_DATASIZE();
1225         insn(conditionalSelect(DATASIZE, true, rm, cond, true, rn, rd));
1226     }
1227
1228     template<int datasize>
1229     ALWAYS_INLINE void eon(RegisterID rd, RegisterID rn, RegisterID rm)
1230     {
1231         eon<datasize>(rd, rn, rm, LSL, 0);
1232     }
1233
1234     template<int datasize>
1235     ALWAYS_INLINE void eon(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)
1236     {
1237         CHECK_DATASIZE();
1238         insn(logicalShiftedRegister(DATASIZE, LogicalOp_EOR, shift, true, rm, amount, rn, rd));
1239     }
1240
1241     template<int datasize>
1242     ALWAYS_INLINE void eor(RegisterID rd, RegisterID rn, RegisterID rm)
1243     {
1244         eor<datasize>(rd, rn, rm, LSL, 0);
1245     }
1246
1247     template<int datasize>
1248     ALWAYS_INLINE void eor(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)
1249     {
1250         CHECK_DATASIZE();
1251         insn(logicalShiftedRegister(DATASIZE, LogicalOp_EOR, shift, false, rm, amount, rn, rd));
1252     }
1253
1254     template<int datasize>
1255     ALWAYS_INLINE void eor(RegisterID rd, RegisterID rn, LogicalImmediate imm)
1256     {
1257         CHECK_DATASIZE();
1258         insn(logicalImmediate(DATASIZE, LogicalOp_EOR, imm.value(), rn, rd));
1259     }
1260
1261     template<int datasize>
1262     ALWAYS_INLINE void extr(RegisterID rd, RegisterID rn, RegisterID rm, int lsb)
1263     {
1264         CHECK_DATASIZE();
1265         insn(extract(DATASIZE, rm, lsb, rn, rd));
1266     }
1267
1268     ALWAYS_INLINE void hint(int imm)
1269     {
1270         insn(hintPseudo(imm));
1271     }
1272
1273     ALWAYS_INLINE void hlt(uint16_t imm)
1274     {
1275         insn(excepnGeneration(ExcepnOp_HALT, imm, 0));
1276     }
1277
1278     template<int datasize>
1279     ALWAYS_INLINE void ldp(RegisterID rt, RegisterID rt2, RegisterID rn, PairPostIndex simm)
1280     {
1281         CHECK_DATASIZE();
1282         insn(loadStoreRegisterPairPostIndex(MEMPAIROPSIZE_INT(datasize), false, MemOp_LOAD, simm, rn, rt, rt2));
1283     }
1284
1285     template<int datasize>
1286     ALWAYS_INLINE void ldp(RegisterID rt, RegisterID rt2, RegisterID rn, PairPreIndex simm)
1287     {
1288         CHECK_DATASIZE();
1289         insn(loadStoreRegisterPairPreIndex(MEMPAIROPSIZE_INT(datasize), false, MemOp_LOAD, simm, rn, rt, rt2));
1290     }
1291
1292     template<int datasize>
1293     ALWAYS_INLINE void ldr(RegisterID rt, RegisterID rn, RegisterID rm)
1294     {
1295         ldr<datasize>(rt, rn, rm, UXTX, 0);
1296     }
1297
1298     template<int datasize>
1299     ALWAYS_INLINE void ldr(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1300     {
1301         CHECK_DATASIZE();
1302         insn(loadStoreRegisterRegisterOffset(MEMOPSIZE, false, MemOp_LOAD, rm, extend, encodeShiftAmount<datasize>(amount), rn, rt));
1303     }
1304
1305     template<int datasize>
1306     ALWAYS_INLINE void ldr(RegisterID rt, RegisterID rn, unsigned pimm)
1307     {
1308         CHECK_DATASIZE();
1309         insn(loadStoreRegisterUnsignedImmediate(MEMOPSIZE, false, MemOp_LOAD, encodePositiveImmediate<datasize>(pimm), rn, rt));
1310     }
1311
1312     template<int datasize>
1313     ALWAYS_INLINE void ldr(RegisterID rt, RegisterID rn, PostIndex simm)
1314     {
1315         CHECK_DATASIZE();
1316         insn(loadStoreRegisterPostIndex(MEMOPSIZE, false, MemOp_LOAD, simm, rn, rt));
1317     }
1318
1319     template<int datasize>
1320     ALWAYS_INLINE void ldr(RegisterID rt, RegisterID rn, PreIndex simm)
1321     {
1322         CHECK_DATASIZE();
1323         insn(loadStoreRegisterPreIndex(MEMOPSIZE, false, MemOp_LOAD, simm, rn, rt));
1324     }
1325
1326     template<int datasize>
1327     ALWAYS_INLINE void ldr_literal(RegisterID rt, int offset = 0)
1328     {
1329         CHECK_DATASIZE();
1330         ASSERT(!(offset & 3));
1331         insn(loadRegisterLiteral(datasize == 64 ? LdrLiteralOp_64BIT : LdrLiteralOp_32BIT, false, offset >> 2, rt));
1332     }
1333
1334     ALWAYS_INLINE void ldrb(RegisterID rt, RegisterID rn, RegisterID rm)
1335     {
1336         // Not calling the 5 argument form of ldrb, since is amount is ommitted S is false.
1337         insn(loadStoreRegisterRegisterOffset(MemOpSize_8_or_128, false, MemOp_LOAD, rm, UXTX, false, rn, rt));
1338     }
1339
1340     ALWAYS_INLINE void ldrb(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1341     {
1342         ASSERT_UNUSED(amount, !amount);
1343         insn(loadStoreRegisterRegisterOffset(MemOpSize_8_or_128, false, MemOp_LOAD, rm, extend, true, rn, rt));
1344     }
1345
1346     ALWAYS_INLINE void ldrb(RegisterID rt, RegisterID rn, unsigned pimm)
1347     {
1348         insn(loadStoreRegisterUnsignedImmediate(MemOpSize_8_or_128, false, MemOp_LOAD, encodePositiveImmediate<8>(pimm), rn, rt));
1349     }
1350
1351     ALWAYS_INLINE void ldrb(RegisterID rt, RegisterID rn, PostIndex simm)
1352     {
1353         insn(loadStoreRegisterPostIndex(MemOpSize_8_or_128, false, MemOp_LOAD, simm, rn, rt));
1354     }
1355
1356     ALWAYS_INLINE void ldrb(RegisterID rt, RegisterID rn, PreIndex simm)
1357     {
1358         insn(loadStoreRegisterPreIndex(MemOpSize_8_or_128, false, MemOp_LOAD, simm, rn, rt));
1359     }
1360
1361     ALWAYS_INLINE void ldrh(RegisterID rt, RegisterID rn, RegisterID rm)
1362     {
1363         ldrh(rt, rn, rm, UXTX, 0);
1364     }
1365
1366     ALWAYS_INLINE void ldrh(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1367     {
1368         ASSERT(!amount || amount == 1);
1369         insn(loadStoreRegisterRegisterOffset(MemOpSize_16, false, MemOp_LOAD, rm, extend, amount == 1, rn, rt));
1370     }
1371
1372     ALWAYS_INLINE void ldrh(RegisterID rt, RegisterID rn, unsigned pimm)
1373     {
1374         insn(loadStoreRegisterUnsignedImmediate(MemOpSize_16, false, MemOp_LOAD, encodePositiveImmediate<16>(pimm), rn, rt));
1375     }
1376
1377     ALWAYS_INLINE void ldrh(RegisterID rt, RegisterID rn, PostIndex simm)
1378     {
1379         insn(loadStoreRegisterPostIndex(MemOpSize_16, false, MemOp_LOAD, simm, rn, rt));
1380     }
1381
1382     ALWAYS_INLINE void ldrh(RegisterID rt, RegisterID rn, PreIndex simm)
1383     {
1384         insn(loadStoreRegisterPreIndex(MemOpSize_16, false, MemOp_LOAD, simm, rn, rt));
1385     }
1386
1387     template<int datasize>
1388     ALWAYS_INLINE void ldrsb(RegisterID rt, RegisterID rn, RegisterID rm)
1389     {
1390         CHECK_DATASIZE();
1391         // Not calling the 5 argument form of ldrsb, since is amount is ommitted S is false.
1392         insn(loadStoreRegisterRegisterOffset(MemOpSize_8_or_128, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, rm, UXTX, false, rn, rt));
1393     }
1394
1395     template<int datasize>
1396     ALWAYS_INLINE void ldrsb(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1397     {
1398         CHECK_DATASIZE();
1399         ASSERT_UNUSED(amount, !amount);
1400         insn(loadStoreRegisterRegisterOffset(MemOpSize_8_or_128, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, rm, extend, true, rn, rt));
1401     }
1402
1403     template<int datasize>
1404     ALWAYS_INLINE void ldrsb(RegisterID rt, RegisterID rn, unsigned pimm)
1405     {
1406         CHECK_DATASIZE();
1407         insn(loadStoreRegisterUnsignedImmediate(MemOpSize_8_or_128, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, encodePositiveImmediate<8>(pimm), rn, rt));
1408     }
1409
1410     template<int datasize>
1411     ALWAYS_INLINE void ldrsb(RegisterID rt, RegisterID rn, PostIndex simm)
1412     {
1413         CHECK_DATASIZE();
1414         insn(loadStoreRegisterPostIndex(MemOpSize_8_or_128, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, simm, rn, rt));
1415     }
1416
1417     template<int datasize>
1418     ALWAYS_INLINE void ldrsb(RegisterID rt, RegisterID rn, PreIndex simm)
1419     {
1420         CHECK_DATASIZE();
1421         insn(loadStoreRegisterPreIndex(MemOpSize_8_or_128, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, simm, rn, rt));
1422     }
1423
1424     template<int datasize>
1425     ALWAYS_INLINE void ldrsh(RegisterID rt, RegisterID rn, RegisterID rm)
1426     {
1427         ldrsh<datasize>(rt, rn, rm, UXTX, 0);
1428     }
1429
1430     template<int datasize>
1431     ALWAYS_INLINE void ldrsh(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1432     {
1433         CHECK_DATASIZE();
1434         ASSERT(!amount || amount == 1);
1435         insn(loadStoreRegisterRegisterOffset(MemOpSize_16, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, rm, extend, amount == 1, rn, rt));
1436     }
1437
1438     template<int datasize>
1439     ALWAYS_INLINE void ldrsh(RegisterID rt, RegisterID rn, unsigned pimm)
1440     {
1441         CHECK_DATASIZE();
1442         insn(loadStoreRegisterUnsignedImmediate(MemOpSize_16, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, encodePositiveImmediate<16>(pimm), rn, rt));
1443     }
1444
1445     template<int datasize>
1446     ALWAYS_INLINE void ldrsh(RegisterID rt, RegisterID rn, PostIndex simm)
1447     {
1448         CHECK_DATASIZE();
1449         insn(loadStoreRegisterPostIndex(MemOpSize_16, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, simm, rn, rt));
1450     }
1451
1452     template<int datasize>
1453     ALWAYS_INLINE void ldrsh(RegisterID rt, RegisterID rn, PreIndex simm)
1454     {
1455         CHECK_DATASIZE();
1456         insn(loadStoreRegisterPreIndex(MemOpSize_16, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, simm, rn, rt));
1457     }
1458
1459     ALWAYS_INLINE void ldrsw(RegisterID rt, RegisterID rn, RegisterID rm)
1460     {
1461         ldrsw(rt, rn, rm, UXTX, 0);
1462     }
1463
1464     ALWAYS_INLINE void ldrsw(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1465     {
1466         ASSERT(!amount || amount == 2);
1467         insn(loadStoreRegisterRegisterOffset(MemOpSize_32, false, MemOp_LOAD_signed64, rm, extend, amount == 2, rn, rt));
1468     }
1469
1470     ALWAYS_INLINE void ldrsw(RegisterID rt, RegisterID rn, unsigned pimm)
1471     {
1472         insn(loadStoreRegisterUnsignedImmediate(MemOpSize_32, false, MemOp_LOAD_signed64, encodePositiveImmediate<32>(pimm), rn, rt));
1473     }
1474
1475     ALWAYS_INLINE void ldrsw(RegisterID rt, RegisterID rn, PostIndex simm)
1476     {
1477         insn(loadStoreRegisterPostIndex(MemOpSize_32, false, MemOp_LOAD_signed64, simm, rn, rt));
1478     }
1479
1480     ALWAYS_INLINE void ldrsw(RegisterID rt, RegisterID rn, PreIndex simm)
1481     {
1482         insn(loadStoreRegisterPreIndex(MemOpSize_32, false, MemOp_LOAD_signed64, simm, rn, rt));
1483     }
1484
1485     ALWAYS_INLINE void ldrsw_literal(RegisterID rt, int offset = 0)
1486     {
1487         ASSERT(!(offset & 3));
1488         insn(loadRegisterLiteral(LdrLiteralOp_LDRSW, false, offset >> 2, rt));
1489     }
1490
1491     template<int datasize>
1492     ALWAYS_INLINE void ldur(RegisterID rt, RegisterID rn, int simm)
1493     {
1494         CHECK_DATASIZE();
1495         insn(loadStoreRegisterUnscaledImmediate(MEMOPSIZE, false, MemOp_LOAD, simm, rn, rt));
1496     }
1497
1498     ALWAYS_INLINE void ldurb(RegisterID rt, RegisterID rn, int simm)
1499     {
1500         insn(loadStoreRegisterUnscaledImmediate(MemOpSize_8_or_128, false, MemOp_LOAD, simm, rn, rt));
1501     }
1502
1503     ALWAYS_INLINE void ldurh(RegisterID rt, RegisterID rn, int simm)
1504     {
1505         insn(loadStoreRegisterUnscaledImmediate(MemOpSize_16, false, MemOp_LOAD, simm, rn, rt));
1506     }
1507
1508     template<int datasize>
1509     ALWAYS_INLINE void ldursb(RegisterID rt, RegisterID rn, int simm)
1510     {
1511         CHECK_DATASIZE();
1512         insn(loadStoreRegisterUnscaledImmediate(MemOpSize_8_or_128, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, simm, rn, rt));
1513     }
1514
1515     template<int datasize>
1516     ALWAYS_INLINE void ldursh(RegisterID rt, RegisterID rn, int simm)
1517     {
1518         CHECK_DATASIZE();
1519         insn(loadStoreRegisterUnscaledImmediate(MemOpSize_16, false, (datasize == 64) ? MemOp_LOAD_signed64 : MemOp_LOAD_signed32, simm, rn, rt));
1520     }
1521
1522     ALWAYS_INLINE void ldursw(RegisterID rt, RegisterID rn, int simm)
1523     {
1524         insn(loadStoreRegisterUnscaledImmediate(MemOpSize_32, false, MemOp_LOAD_signed64, simm, rn, rt));
1525     }
1526
1527     template<int datasize>
1528     ALWAYS_INLINE void lsl(RegisterID rd, RegisterID rn, int shift)
1529     {
1530         ASSERT(shift < datasize);
1531         ubfm<datasize>(rd, rn, (datasize - shift) & (datasize - 1), datasize - 1 - shift);
1532     }
1533
1534     template<int datasize>
1535     ALWAYS_INLINE void lsl(RegisterID rd, RegisterID rn, RegisterID rm)
1536     {
1537         lslv<datasize>(rd, rn, rm);
1538     }
1539
1540     template<int datasize>
1541     ALWAYS_INLINE void lslv(RegisterID rd, RegisterID rn, RegisterID rm)
1542     {
1543         CHECK_DATASIZE();
1544         insn(dataProcessing2Source(DATASIZE, rm, DataOp_LSLV, rn, rd));
1545     }
1546
1547     template<int datasize>
1548     ALWAYS_INLINE void lsr(RegisterID rd, RegisterID rn, int shift)
1549     {
1550         ASSERT(shift < datasize);
1551         ubfm<datasize>(rd, rn, shift, datasize - 1);
1552     }
1553
1554     template<int datasize>
1555     ALWAYS_INLINE void lsr(RegisterID rd, RegisterID rn, RegisterID rm)
1556     {
1557         lsrv<datasize>(rd, rn, rm);
1558     }
1559
1560     template<int datasize>
1561     ALWAYS_INLINE void lsrv(RegisterID rd, RegisterID rn, RegisterID rm)
1562     {
1563         CHECK_DATASIZE();
1564         insn(dataProcessing2Source(DATASIZE, rm, DataOp_LSRV, rn, rd));
1565     }
1566
1567     template<int datasize>
1568     ALWAYS_INLINE void madd(RegisterID rd, RegisterID rn, RegisterID rm, RegisterID ra)
1569     {
1570         CHECK_DATASIZE();
1571         insn(dataProcessing3Source(DATASIZE, DataOp_MADD, rm, ra, rn, rd));
1572     }
1573
1574     template<int datasize>
1575     ALWAYS_INLINE void mneg(RegisterID rd, RegisterID rn, RegisterID rm)
1576     {
1577         msub<datasize>(rd, rn, rm, ARM64Registers::zr);
1578     }
1579
1580     template<int datasize>
1581     ALWAYS_INLINE void mov(RegisterID rd, RegisterID rm)
1582     {
1583         if (isSp(rd) || isSp(rm))
1584             add<datasize>(rd, rm, UInt12(0));
1585         else
1586             orr<datasize>(rd, ARM64Registers::zr, rm);
1587     }
1588
1589     template<int datasize>
1590     ALWAYS_INLINE void movi(RegisterID rd, LogicalImmediate imm)
1591     {
1592         orr<datasize>(rd, ARM64Registers::zr, imm);
1593     }
1594
1595     template<int datasize>
1596     ALWAYS_INLINE void movk(RegisterID rd, uint16_t value, int shift = 0)
1597     {
1598         CHECK_DATASIZE();
1599         ASSERT(!(shift & 0xf));
1600         insn(moveWideImediate(DATASIZE, MoveWideOp_K, shift >> 4, value, rd));
1601     }
1602
1603     template<int datasize>
1604     ALWAYS_INLINE void movn(RegisterID rd, uint16_t value, int shift = 0)
1605     {
1606         CHECK_DATASIZE();
1607         ASSERT(!(shift & 0xf));
1608         insn(moveWideImediate(DATASIZE, MoveWideOp_N, shift >> 4, value, rd));
1609     }
1610
1611     template<int datasize>
1612     ALWAYS_INLINE void movz(RegisterID rd, uint16_t value, int shift = 0)
1613     {
1614         CHECK_DATASIZE();
1615         ASSERT(!(shift & 0xf));
1616         insn(moveWideImediate(DATASIZE, MoveWideOp_Z, shift >> 4, value, rd));
1617     }
1618
1619     template<int datasize>
1620     ALWAYS_INLINE void msub(RegisterID rd, RegisterID rn, RegisterID rm, RegisterID ra)
1621     {
1622         CHECK_DATASIZE();
1623         insn(dataProcessing3Source(DATASIZE, DataOp_MSUB, rm, ra, rn, rd));
1624     }
1625
1626     template<int datasize>
1627     ALWAYS_INLINE void mul(RegisterID rd, RegisterID rn, RegisterID rm)
1628     {
1629         madd<datasize>(rd, rn, rm, ARM64Registers::zr);
1630     }
1631
1632     template<int datasize>
1633     ALWAYS_INLINE void mvn(RegisterID rd, RegisterID rm)
1634     {
1635         orn<datasize>(rd, ARM64Registers::zr, rm);
1636     }
1637
1638     template<int datasize>
1639     ALWAYS_INLINE void mvn(RegisterID rd, RegisterID rm, ShiftType shift, int amount)
1640     {
1641         orn<datasize>(rd, ARM64Registers::zr, rm, shift, amount);
1642     }
1643
1644     template<int datasize, SetFlags setFlags = DontSetFlags>
1645     ALWAYS_INLINE void neg(RegisterID rd, RegisterID rm)
1646     {
1647         sub<datasize, setFlags>(rd, ARM64Registers::zr, rm);
1648     }
1649
1650     template<int datasize, SetFlags setFlags = DontSetFlags>
1651     ALWAYS_INLINE void neg(RegisterID rd, RegisterID rm, ShiftType shift, int amount)
1652     {
1653         sub<datasize, setFlags>(rd, ARM64Registers::zr, rm, shift, amount);
1654     }
1655
1656     template<int datasize, SetFlags setFlags = DontSetFlags>
1657     ALWAYS_INLINE void ngc(RegisterID rd, RegisterID rm)
1658     {
1659         sbc<datasize, setFlags>(rd, ARM64Registers::zr, rm);
1660     }
1661
1662     template<int datasize, SetFlags setFlags = DontSetFlags>
1663     ALWAYS_INLINE void ngc(RegisterID rd, RegisterID rm, ShiftType shift, int amount)
1664     {
1665         sbc<datasize, setFlags>(rd, ARM64Registers::zr, rm, shift, amount);
1666     }
1667
1668     ALWAYS_INLINE void nop()
1669     {
1670         insn(nopPseudo());
1671     }
1672
1673     static void fillNops(void* base, size_t size)
1674     {
1675         RELEASE_ASSERT(!(size % sizeof(int32_t)));
1676         size_t n = size / sizeof(int32_t);
1677         for (int32_t* ptr = static_cast<int32_t*>(base); n--;)
1678             *ptr++ = nopPseudo();
1679     }
1680
1681     ALWAYS_INLINE void dmbSY()
1682     {
1683         insn(0xd5033fbf);
1684     }
1685
1686     template<int datasize>
1687     ALWAYS_INLINE void orn(RegisterID rd, RegisterID rn, RegisterID rm)
1688     {
1689         orn<datasize>(rd, rn, rm, LSL, 0);
1690     }
1691
1692     template<int datasize>
1693     ALWAYS_INLINE void orn(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)
1694     {
1695         CHECK_DATASIZE();
1696         insn(logicalShiftedRegister(DATASIZE, LogicalOp_ORR, shift, true, rm, amount, rn, rd));
1697     }
1698
1699     template<int datasize>
1700     ALWAYS_INLINE void orr(RegisterID rd, RegisterID rn, RegisterID rm)
1701     {
1702         orr<datasize>(rd, rn, rm, LSL, 0);
1703     }
1704
1705     template<int datasize>
1706     ALWAYS_INLINE void orr(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)
1707     {
1708         CHECK_DATASIZE();
1709         insn(logicalShiftedRegister(DATASIZE, LogicalOp_ORR, shift, false, rm, amount, rn, rd));
1710     }
1711
1712     template<int datasize>
1713     ALWAYS_INLINE void orr(RegisterID rd, RegisterID rn, LogicalImmediate imm)
1714     {
1715         CHECK_DATASIZE();
1716         insn(logicalImmediate(DATASIZE, LogicalOp_ORR, imm.value(), rn, rd));
1717     }
1718
1719     template<int datasize>
1720     ALWAYS_INLINE void rbit(RegisterID rd, RegisterID rn)
1721     {
1722         CHECK_DATASIZE();
1723         insn(dataProcessing1Source(DATASIZE, DataOp_RBIT, rn, rd));
1724     }
1725
1726     ALWAYS_INLINE void ret(RegisterID rn = ARM64Registers::lr)
1727     {
1728         insn(unconditionalBranchRegister(BranchType_RET, rn));
1729     }
1730
1731     template<int datasize>
1732     ALWAYS_INLINE void rev(RegisterID rd, RegisterID rn)
1733     {
1734         CHECK_DATASIZE();
1735         if (datasize == 32) // 'rev' mnemonic means REV32 or REV64 depending on the operand width.
1736             insn(dataProcessing1Source(Datasize_32, DataOp_REV32, rn, rd));
1737         else
1738             insn(dataProcessing1Source(Datasize_64, DataOp_REV64, rn, rd));
1739     }
1740
1741     template<int datasize>
1742     ALWAYS_INLINE void rev16(RegisterID rd, RegisterID rn)
1743     {
1744         CHECK_DATASIZE();
1745         insn(dataProcessing1Source(DATASIZE, DataOp_REV16, rn, rd));
1746     }
1747
1748     template<int datasize>
1749     ALWAYS_INLINE void rev32(RegisterID rd, RegisterID rn)
1750     {
1751         ASSERT(datasize == 64); // 'rev32' only valid with 64-bit operands.
1752         insn(dataProcessing1Source(Datasize_64, DataOp_REV32, rn, rd));
1753     }
1754
1755     template<int datasize>
1756     ALWAYS_INLINE void ror(RegisterID rd, RegisterID rn, RegisterID rm)
1757     {
1758         rorv<datasize>(rd, rn, rm);
1759     }
1760
1761     template<int datasize>
1762     ALWAYS_INLINE void ror(RegisterID rd, RegisterID rs, int shift)
1763     {
1764         extr<datasize>(rd, rs, rs, shift);
1765     }
1766
1767     template<int datasize>
1768     ALWAYS_INLINE void rorv(RegisterID rd, RegisterID rn, RegisterID rm)
1769     {
1770         CHECK_DATASIZE();
1771         insn(dataProcessing2Source(DATASIZE, rm, DataOp_RORV, rn, rd));
1772     }
1773
1774     template<int datasize, SetFlags setFlags = DontSetFlags>
1775     ALWAYS_INLINE void sbc(RegisterID rd, RegisterID rn, RegisterID rm)
1776     {
1777         CHECK_DATASIZE();
1778         insn(addSubtractWithCarry(DATASIZE, AddOp_SUB, setFlags, rm, rn, rd));
1779     }
1780
1781     template<int datasize>
1782     ALWAYS_INLINE void sbfiz(RegisterID rd, RegisterID rn, int lsb, int width)
1783     {
1784         sbfm<datasize>(rd, rn, (datasize - lsb) & (datasize - 1), width - 1);
1785     }
1786
1787     template<int datasize>
1788     ALWAYS_INLINE void sbfm(RegisterID rd, RegisterID rn, int immr, int imms)
1789     {
1790         CHECK_DATASIZE();
1791         insn(bitfield(DATASIZE, BitfieldOp_SBFM, immr, imms, rn, rd));
1792     }
1793
1794     template<int datasize>
1795     ALWAYS_INLINE void sbfx(RegisterID rd, RegisterID rn, int lsb, int width)
1796     {
1797         sbfm<datasize>(rd, rn, lsb, lsb + width - 1);
1798     }
1799
1800     template<int datasize>
1801     ALWAYS_INLINE void sdiv(RegisterID rd, RegisterID rn, RegisterID rm)
1802     {
1803         CHECK_DATASIZE();
1804         insn(dataProcessing2Source(DATASIZE, rm, DataOp_SDIV, rn, rd));
1805     }
1806
1807     ALWAYS_INLINE void smaddl(RegisterID rd, RegisterID rn, RegisterID rm, RegisterID ra)
1808     {
1809         insn(dataProcessing3Source(Datasize_64, DataOp_SMADDL, rm, ra, rn, rd));
1810     }
1811
1812     ALWAYS_INLINE void smnegl(RegisterID rd, RegisterID rn, RegisterID rm)
1813     {
1814         smsubl(rd, rn, rm, ARM64Registers::zr);
1815     }
1816
1817     ALWAYS_INLINE void smsubl(RegisterID rd, RegisterID rn, RegisterID rm, RegisterID ra)
1818     {
1819         insn(dataProcessing3Source(Datasize_64, DataOp_SMSUBL, rm, ra, rn, rd));
1820     }
1821
1822     ALWAYS_INLINE void smulh(RegisterID rd, RegisterID rn, RegisterID rm)
1823     {
1824         insn(dataProcessing3Source(Datasize_64, DataOp_SMULH, rm, ARM64Registers::zr, rn, rd));
1825     }
1826
1827     ALWAYS_INLINE void smull(RegisterID rd, RegisterID rn, RegisterID rm)
1828     {
1829         smaddl(rd, rn, rm, ARM64Registers::zr);
1830     }
1831
1832     template<int datasize>
1833     ALWAYS_INLINE void stp(RegisterID rt, RegisterID rt2, RegisterID rn, PairPostIndex simm)
1834     {
1835         CHECK_DATASIZE();
1836         insn(loadStoreRegisterPairPostIndex(MEMPAIROPSIZE_INT(datasize), false, MemOp_STORE, simm, rn, rt, rt2));
1837     }
1838
1839     template<int datasize>
1840     ALWAYS_INLINE void stp(RegisterID rt, RegisterID rt2, RegisterID rn, PairPreIndex simm)
1841     {
1842         CHECK_DATASIZE();
1843         insn(loadStoreRegisterPairPreIndex(MEMPAIROPSIZE_INT(datasize), false, MemOp_STORE, simm, rn, rt, rt2));
1844     }
1845
1846     template<int datasize>
1847     ALWAYS_INLINE void str(RegisterID rt, RegisterID rn, RegisterID rm)
1848     {
1849         str<datasize>(rt, rn, rm, UXTX, 0);
1850     }
1851
1852     template<int datasize>
1853     ALWAYS_INLINE void str(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1854     {
1855         CHECK_DATASIZE();
1856         insn(loadStoreRegisterRegisterOffset(MEMOPSIZE, false, MemOp_STORE, rm, extend, encodeShiftAmount<datasize>(amount), rn, rt));
1857     }
1858
1859     template<int datasize>
1860     ALWAYS_INLINE void str(RegisterID rt, RegisterID rn, unsigned pimm)
1861     {
1862         CHECK_DATASIZE();
1863         insn(loadStoreRegisterUnsignedImmediate(MEMOPSIZE, false, MemOp_STORE, encodePositiveImmediate<datasize>(pimm), rn, rt));
1864     }
1865
1866     template<int datasize>
1867     ALWAYS_INLINE void str(RegisterID rt, RegisterID rn, PostIndex simm)
1868     {
1869         CHECK_DATASIZE();
1870         insn(loadStoreRegisterPostIndex(MEMOPSIZE, false, MemOp_STORE, simm, rn, rt));
1871     }
1872
1873     template<int datasize>
1874     ALWAYS_INLINE void str(RegisterID rt, RegisterID rn, PreIndex simm)
1875     {
1876         CHECK_DATASIZE();
1877         insn(loadStoreRegisterPreIndex(MEMOPSIZE, false, MemOp_STORE, simm, rn, rt));
1878     }
1879
1880     ALWAYS_INLINE void strb(RegisterID rt, RegisterID rn, RegisterID rm)
1881     {
1882         // Not calling the 5 argument form of strb, since is amount is ommitted S is false.
1883         insn(loadStoreRegisterRegisterOffset(MemOpSize_8_or_128, false, MemOp_STORE, rm, UXTX, false, rn, rt));
1884     }
1885
1886     ALWAYS_INLINE void strb(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1887     {
1888         ASSERT_UNUSED(amount, !amount);
1889         insn(loadStoreRegisterRegisterOffset(MemOpSize_8_or_128, false, MemOp_STORE, rm, extend, true, rn, rt));
1890     }
1891
1892     ALWAYS_INLINE void strb(RegisterID rt, RegisterID rn, unsigned pimm)
1893     {
1894         insn(loadStoreRegisterUnsignedImmediate(MemOpSize_8_or_128, false, MemOp_STORE, encodePositiveImmediate<8>(pimm), rn, rt));
1895     }
1896
1897     ALWAYS_INLINE void strb(RegisterID rt, RegisterID rn, PostIndex simm)
1898     {
1899         insn(loadStoreRegisterPostIndex(MemOpSize_8_or_128, false, MemOp_STORE, simm, rn, rt));
1900     }
1901
1902     ALWAYS_INLINE void strb(RegisterID rt, RegisterID rn, PreIndex simm)
1903     {
1904         insn(loadStoreRegisterPreIndex(MemOpSize_8_or_128, false, MemOp_STORE, simm, rn, rt));
1905     }
1906
1907     ALWAYS_INLINE void strh(RegisterID rt, RegisterID rn, RegisterID rm)
1908     {
1909         strh(rt, rn, rm, UXTX, 0);
1910     }
1911
1912     ALWAYS_INLINE void strh(RegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1913     {
1914         ASSERT(!amount || amount == 1);
1915         insn(loadStoreRegisterRegisterOffset(MemOpSize_16, false, MemOp_STORE, rm, extend, amount == 1, rn, rt));
1916     }
1917
1918     ALWAYS_INLINE void strh(RegisterID rt, RegisterID rn, unsigned pimm)
1919     {
1920         insn(loadStoreRegisterUnsignedImmediate(MemOpSize_16, false, MemOp_STORE, encodePositiveImmediate<16>(pimm), rn, rt));
1921     }
1922
1923     ALWAYS_INLINE void strh(RegisterID rt, RegisterID rn, PostIndex simm)
1924     {
1925         insn(loadStoreRegisterPostIndex(MemOpSize_16, false, MemOp_STORE, simm, rn, rt));
1926     }
1927
1928     ALWAYS_INLINE void strh(RegisterID rt, RegisterID rn, PreIndex simm)
1929     {
1930         insn(loadStoreRegisterPreIndex(MemOpSize_16, false, MemOp_STORE, simm, rn, rt));
1931     }
1932
1933     template<int datasize>
1934     ALWAYS_INLINE void stur(RegisterID rt, RegisterID rn, int simm)
1935     {
1936         CHECK_DATASIZE();
1937         insn(loadStoreRegisterUnscaledImmediate(MEMOPSIZE, false, MemOp_STORE, simm, rn, rt));
1938     }
1939
1940     ALWAYS_INLINE void sturb(RegisterID rt, RegisterID rn, int simm)
1941     {
1942         insn(loadStoreRegisterUnscaledImmediate(MemOpSize_8_or_128, false, MemOp_STORE, simm, rn, rt));
1943     }
1944
1945     ALWAYS_INLINE void sturh(RegisterID rt, RegisterID rn, int simm)
1946     {
1947         insn(loadStoreRegisterUnscaledImmediate(MemOpSize_16, false, MemOp_STORE, simm, rn, rt));
1948     }
1949
1950     template<int datasize, SetFlags setFlags = DontSetFlags>
1951     ALWAYS_INLINE void sub(RegisterID rd, RegisterID rn, UInt12 imm12, int shift = 0)
1952     {
1953         CHECK_DATASIZE();
1954         ASSERT(!shift || shift == 12);
1955         insn(addSubtractImmediate(DATASIZE, AddOp_SUB, setFlags, shift == 12, imm12, rn, rd));
1956     }
1957
1958     template<int datasize, SetFlags setFlags = DontSetFlags>
1959     ALWAYS_INLINE void sub(RegisterID rd, RegisterID rn, RegisterID rm)
1960     {
1961         sub<datasize, setFlags>(rd, rn, rm, LSL, 0);
1962     }
1963
1964     template<int datasize, SetFlags setFlags = DontSetFlags>
1965     ALWAYS_INLINE void sub(RegisterID rd, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
1966     {
1967         CHECK_DATASIZE();
1968         insn(addSubtractExtendedRegister(DATASIZE, AddOp_SUB, setFlags, rm, extend, amount, rn, rd));
1969     }
1970
1971     template<int datasize, SetFlags setFlags = DontSetFlags>
1972     ALWAYS_INLINE void sub(RegisterID rd, RegisterID rn, RegisterID rm, ShiftType shift, int amount)
1973     {
1974         CHECK_DATASIZE();
1975         if (isSp(rd) || isSp(rn)) {
1976             ASSERT(shift == LSL);
1977             ASSERT(!isSp(rm));
1978             sub<datasize, setFlags>(rd, rn, rm, UXTX, amount);
1979         } else
1980             insn(addSubtractShiftedRegister(DATASIZE, AddOp_SUB, setFlags, shift, rm, amount, rn, rd));
1981     }
1982
1983     template<int datasize>
1984     ALWAYS_INLINE void sxtb(RegisterID rd, RegisterID rn)
1985     {
1986         sbfm<datasize>(rd, rn, 0, 7);
1987     }
1988
1989     template<int datasize>
1990     ALWAYS_INLINE void sxth(RegisterID rd, RegisterID rn)
1991     {
1992         sbfm<datasize>(rd, rn, 0, 15);
1993     }
1994
1995     ALWAYS_INLINE void sxtw(RegisterID rd, RegisterID rn)
1996     {
1997         sbfm<64>(rd, rn, 0, 31);
1998     }
1999
2000     ALWAYS_INLINE void tbz(RegisterID rt, int imm, int offset = 0)
2001     {
2002         ASSERT(!(offset & 3));
2003         offset >>= 2;
2004         insn(testAndBranchImmediate(false, imm, offset, rt));
2005     }
2006
2007     ALWAYS_INLINE void tbnz(RegisterID rt, int imm, int offset = 0)
2008     {
2009         ASSERT(!(offset & 3));
2010         offset >>= 2;
2011         insn(testAndBranchImmediate(true, imm, offset, rt));
2012     }
2013
2014     template<int datasize>
2015     ALWAYS_INLINE void tst(RegisterID rn, RegisterID rm)
2016     {
2017         and_<datasize, S>(ARM64Registers::zr, rn, rm);
2018     }
2019
2020     template<int datasize>
2021     ALWAYS_INLINE void tst(RegisterID rn, RegisterID rm, ShiftType shift, int amount)
2022     {
2023         and_<datasize, S>(ARM64Registers::zr, rn, rm, shift, amount);
2024     }
2025
2026     template<int datasize>
2027     ALWAYS_INLINE void tst(RegisterID rn, LogicalImmediate imm)
2028     {
2029         and_<datasize, S>(ARM64Registers::zr, rn, imm);
2030     }
2031
2032     template<int datasize>
2033     ALWAYS_INLINE void ubfiz(RegisterID rd, RegisterID rn, int lsb, int width)
2034     {
2035         ubfm<datasize>(rd, rn, (datasize - lsb) & (datasize - 1), width - 1);
2036     }
2037
2038     template<int datasize>
2039     ALWAYS_INLINE void ubfm(RegisterID rd, RegisterID rn, int immr, int imms)
2040     {
2041         CHECK_DATASIZE();
2042         insn(bitfield(DATASIZE, BitfieldOp_UBFM, immr, imms, rn, rd));
2043     }
2044
2045     template<int datasize>
2046     ALWAYS_INLINE void ubfx(RegisterID rd, RegisterID rn, int lsb, int width)
2047     {
2048         ubfm<datasize>(rd, rn, lsb, lsb + width - 1);
2049     }
2050
2051     template<int datasize>
2052     ALWAYS_INLINE void udiv(RegisterID rd, RegisterID rn, RegisterID rm)
2053     {
2054         CHECK_DATASIZE();
2055         insn(dataProcessing2Source(DATASIZE, rm, DataOp_UDIV, rn, rd));
2056     }
2057
2058     ALWAYS_INLINE void umaddl(RegisterID rd, RegisterID rn, RegisterID rm, RegisterID ra)
2059     {
2060         insn(dataProcessing3Source(Datasize_64, DataOp_UMADDL, rm, ra, rn, rd));
2061     }
2062
2063     ALWAYS_INLINE void umnegl(RegisterID rd, RegisterID rn, RegisterID rm)
2064     {
2065         umsubl(rd, rn, rm, ARM64Registers::zr);
2066     }
2067
2068     ALWAYS_INLINE void umsubl(RegisterID rd, RegisterID rn, RegisterID rm, RegisterID ra)
2069     {
2070         insn(dataProcessing3Source(Datasize_64, DataOp_UMSUBL, rm, ra, rn, rd));
2071     }
2072
2073     ALWAYS_INLINE void umulh(RegisterID rd, RegisterID rn, RegisterID rm)
2074     {
2075         insn(dataProcessing3Source(Datasize_64, DataOp_UMULH, rm, ARM64Registers::zr, rn, rd));
2076     }
2077
2078     ALWAYS_INLINE void umull(RegisterID rd, RegisterID rn, RegisterID rm)
2079     {
2080         umaddl(rd, rn, rm, ARM64Registers::zr);
2081     }
2082
2083     template<int datasize>
2084     ALWAYS_INLINE void uxtb(RegisterID rd, RegisterID rn)
2085     {
2086         ubfm<datasize>(rd, rn, 0, 7);
2087     }
2088
2089     template<int datasize>
2090     ALWAYS_INLINE void uxth(RegisterID rd, RegisterID rn)
2091     {
2092         ubfm<datasize>(rd, rn, 0, 15);
2093     }
2094
2095     ALWAYS_INLINE void uxtw(RegisterID rd, RegisterID rn)
2096     {
2097         ubfm<64>(rd, rn, 0, 31);
2098     }
2099
2100     // Floating Point Instructions:
2101
2102     template<int datasize>
2103     ALWAYS_INLINE void fabs(FPRegisterID vd, FPRegisterID vn)
2104     {
2105         CHECK_DATASIZE();
2106         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FABS, vn, vd));
2107     }
2108
2109     template<int datasize>
2110     ALWAYS_INLINE void fadd(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2111     {
2112         CHECK_DATASIZE();
2113         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FADD, vn, vd));
2114     }
2115
2116     template<int datasize>
2117     ALWAYS_INLINE void fccmp(FPRegisterID vn, FPRegisterID vm, int nzcv, Condition cond)
2118     {
2119         CHECK_DATASIZE();
2120         insn(floatingPointConditionalCompare(DATASIZE, vm, cond, vn, FPCondCmpOp_FCMP, nzcv));
2121     }
2122
2123     template<int datasize>
2124     ALWAYS_INLINE void fccmpe(FPRegisterID vn, FPRegisterID vm, int nzcv, Condition cond)
2125     {
2126         CHECK_DATASIZE();
2127         insn(floatingPointConditionalCompare(DATASIZE, vm, cond, vn, FPCondCmpOp_FCMPE, nzcv));
2128     }
2129
2130     template<int datasize>
2131     ALWAYS_INLINE void fcmp(FPRegisterID vn, FPRegisterID vm)
2132     {
2133         CHECK_DATASIZE();
2134         insn(floatingPointCompare(DATASIZE, vm, vn, FPCmpOp_FCMP));
2135     }
2136
2137     template<int datasize>
2138     ALWAYS_INLINE void fcmp_0(FPRegisterID vn)
2139     {
2140         CHECK_DATASIZE();
2141         insn(floatingPointCompare(DATASIZE, static_cast<FPRegisterID>(0), vn, FPCmpOp_FCMP0));
2142     }
2143
2144     template<int datasize>
2145     ALWAYS_INLINE void fcmpe(FPRegisterID vn, FPRegisterID vm)
2146     {
2147         CHECK_DATASIZE();
2148         insn(floatingPointCompare(DATASIZE, vm, vn, FPCmpOp_FCMPE));
2149     }
2150
2151     template<int datasize>
2152     ALWAYS_INLINE void fcmpe_0(FPRegisterID vn)
2153     {
2154         CHECK_DATASIZE();
2155         insn(floatingPointCompare(DATASIZE, static_cast<FPRegisterID>(0), vn, FPCmpOp_FCMPE0));
2156     }
2157
2158     template<int datasize>
2159     ALWAYS_INLINE void fcsel(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm, Condition cond)
2160     {
2161         CHECK_DATASIZE();
2162         insn(floatingPointConditionalSelect(DATASIZE, vm, cond, vn, vd));
2163     }
2164
2165     template<int dstsize, int srcsize>
2166     ALWAYS_INLINE void fcvt(FPRegisterID vd, FPRegisterID vn)
2167     {
2168         ASSERT(dstsize == 16 || dstsize == 32 || dstsize == 64);
2169         ASSERT(srcsize == 16 || srcsize == 32 || srcsize == 64);
2170         ASSERT(dstsize != srcsize);
2171         Datasize type = (srcsize == 64) ? Datasize_64 : (srcsize == 32) ? Datasize_32 : Datasize_16;
2172         FPDataOp1Source opcode = (dstsize == 64) ? FPDataOp_FCVT_toDouble : (dstsize == 32) ? FPDataOp_FCVT_toSingle : FPDataOp_FCVT_toHalf;
2173         insn(floatingPointDataProcessing1Source(type, opcode, vn, vd));
2174     }
2175
2176     template<int dstsize, int srcsize>
2177     ALWAYS_INLINE void fcvtas(RegisterID rd, FPRegisterID vn)
2178     {
2179         CHECK_DATASIZE_OF(dstsize);
2180         CHECK_DATASIZE_OF(srcsize);
2181         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTAS, vn, rd));
2182     }
2183
2184     template<int dstsize, int srcsize>
2185     ALWAYS_INLINE void fcvtau(RegisterID rd, FPRegisterID vn)
2186     {
2187         CHECK_DATASIZE_OF(dstsize);
2188         CHECK_DATASIZE_OF(srcsize);
2189         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTAU, vn, rd));
2190     }
2191
2192     template<int dstsize, int srcsize>
2193     ALWAYS_INLINE void fcvtms(RegisterID rd, FPRegisterID vn)
2194     {
2195         CHECK_DATASIZE_OF(dstsize);
2196         CHECK_DATASIZE_OF(srcsize);
2197         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTMS, vn, rd));
2198     }
2199
2200     template<int dstsize, int srcsize>
2201     ALWAYS_INLINE void fcvtmu(RegisterID rd, FPRegisterID vn)
2202     {
2203         CHECK_DATASIZE_OF(dstsize);
2204         CHECK_DATASIZE_OF(srcsize);
2205         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTMU, vn, rd));
2206     }
2207
2208     template<int dstsize, int srcsize>
2209     ALWAYS_INLINE void fcvtns(RegisterID rd, FPRegisterID vn)
2210     {
2211         CHECK_DATASIZE_OF(dstsize);
2212         CHECK_DATASIZE_OF(srcsize);
2213         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTNS, vn, rd));
2214     }
2215
2216     template<int dstsize, int srcsize>
2217     ALWAYS_INLINE void fcvtnu(RegisterID rd, FPRegisterID vn)
2218     {
2219         CHECK_DATASIZE_OF(dstsize);
2220         CHECK_DATASIZE_OF(srcsize);
2221         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTNU, vn, rd));
2222     }
2223
2224     template<int dstsize, int srcsize>
2225     ALWAYS_INLINE void fcvtps(RegisterID rd, FPRegisterID vn)
2226     {
2227         CHECK_DATASIZE_OF(dstsize);
2228         CHECK_DATASIZE_OF(srcsize);
2229         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTPS, vn, rd));
2230     }
2231
2232     template<int dstsize, int srcsize>
2233     ALWAYS_INLINE void fcvtpu(RegisterID rd, FPRegisterID vn)
2234     {
2235         CHECK_DATASIZE_OF(dstsize);
2236         CHECK_DATASIZE_OF(srcsize);
2237         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTPU, vn, rd));
2238     }
2239
2240     template<int dstsize, int srcsize>
2241     ALWAYS_INLINE void fcvtzs(RegisterID rd, FPRegisterID vn)
2242     {
2243         CHECK_DATASIZE_OF(dstsize);
2244         CHECK_DATASIZE_OF(srcsize);
2245         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTZS, vn, rd));
2246     }
2247
2248     template<int dstsize, int srcsize>
2249     ALWAYS_INLINE void fcvtzu(RegisterID rd, FPRegisterID vn)
2250     {
2251         CHECK_DATASIZE_OF(dstsize);
2252         CHECK_DATASIZE_OF(srcsize);
2253         insn(floatingPointIntegerConversions(DATASIZE_OF(dstsize), DATASIZE_OF(srcsize), FPIntConvOp_FCVTZU, vn, rd));
2254     }
2255
2256     template<int datasize>
2257     ALWAYS_INLINE void fdiv(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2258     {
2259         CHECK_DATASIZE();
2260         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FDIV, vn, vd));
2261     }
2262
2263     template<int datasize>
2264     ALWAYS_INLINE void fmadd(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm, FPRegisterID va)
2265     {
2266         CHECK_DATASIZE();
2267         insn(floatingPointDataProcessing3Source(DATASIZE, false, vm, AddOp_ADD, va, vn, vd));
2268     }
2269
2270     template<int datasize>
2271     ALWAYS_INLINE void fmax(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2272     {
2273         CHECK_DATASIZE();
2274         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FMAX, vn, vd));
2275     }
2276
2277     template<int datasize>
2278     ALWAYS_INLINE void fmaxnm(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2279     {
2280         CHECK_DATASIZE();
2281         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FMAXNM, vn, vd));
2282     }
2283
2284     template<int datasize>
2285     ALWAYS_INLINE void fmin(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2286     {
2287         CHECK_DATASIZE();
2288         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FMIN, vn, vd));
2289     }
2290
2291     template<int datasize>
2292     ALWAYS_INLINE void fminnm(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2293     {
2294         CHECK_DATASIZE();
2295         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FMINNM, vn, vd));
2296     }
2297
2298     template<int datasize>
2299     ALWAYS_INLINE void fmov(FPRegisterID vd, FPRegisterID vn)
2300     {
2301         CHECK_DATASIZE();
2302         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FMOV, vn, vd));
2303     }
2304
2305     template<int datasize>
2306     ALWAYS_INLINE void fmov(FPRegisterID vd, RegisterID rn)
2307     {
2308         CHECK_DATASIZE();
2309         insn(floatingPointIntegerConversions(DATASIZE, DATASIZE, FPIntConvOp_FMOV_XtoQ, rn, vd));
2310     }
2311
2312     template<int datasize>
2313     ALWAYS_INLINE void fmov(RegisterID rd, FPRegisterID vn)
2314     {
2315         CHECK_DATASIZE();
2316         insn(floatingPointIntegerConversions(DATASIZE, DATASIZE, FPIntConvOp_FMOV_QtoX, vn, rd));
2317     }
2318
2319     template<int datasize>
2320     ALWAYS_INLINE void fmov(FPRegisterID vd, double imm)
2321     {
2322         CHECK_DATASIZE();
2323         insn(floatingPointImmediate(DATASIZE, encodeFPImm(imm), vd));
2324     }
2325
2326     ALWAYS_INLINE void fmov_top(FPRegisterID vd, RegisterID rn)
2327     {
2328         insn(floatingPointIntegerConversions(Datasize_64, Datasize_64, FPIntConvOp_FMOV_XtoQ_top, rn, vd));
2329     }
2330
2331     ALWAYS_INLINE void fmov_top(RegisterID rd, FPRegisterID vn)
2332     {
2333         insn(floatingPointIntegerConversions(Datasize_64, Datasize_64, FPIntConvOp_FMOV_QtoX_top, vn, rd));
2334     }
2335
2336     template<int datasize>
2337     ALWAYS_INLINE void fmsub(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm, FPRegisterID va)
2338     {
2339         CHECK_DATASIZE();
2340         insn(floatingPointDataProcessing3Source(DATASIZE, false, vm, AddOp_SUB, va, vn, vd));
2341     }
2342
2343     template<int datasize>
2344     ALWAYS_INLINE void fmul(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2345     {
2346         CHECK_DATASIZE();
2347         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FMUL, vn, vd));
2348     }
2349
2350     template<int datasize>
2351     ALWAYS_INLINE void fneg(FPRegisterID vd, FPRegisterID vn)
2352     {
2353         CHECK_DATASIZE();
2354         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FNEG, vn, vd));
2355     }
2356
2357     template<int datasize>
2358     ALWAYS_INLINE void fnmadd(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm, FPRegisterID va)
2359     {
2360         CHECK_DATASIZE();
2361         insn(floatingPointDataProcessing3Source(DATASIZE, true, vm, AddOp_ADD, va, vn, vd));
2362     }
2363
2364     template<int datasize>
2365     ALWAYS_INLINE void fnmsub(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm, FPRegisterID va)
2366     {
2367         CHECK_DATASIZE();
2368         insn(floatingPointDataProcessing3Source(DATASIZE, true, vm, AddOp_SUB, va, vn, vd));
2369     }
2370
2371     template<int datasize>
2372     ALWAYS_INLINE void fnmul(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2373     {
2374         CHECK_DATASIZE();
2375         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FNMUL, vn, vd));
2376     }
2377
2378     template<int datasize>
2379     ALWAYS_INLINE void frinta(FPRegisterID vd, FPRegisterID vn)
2380     {
2381         CHECK_DATASIZE();
2382         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FRINTA, vn, vd));
2383     }
2384
2385     template<int datasize>
2386     ALWAYS_INLINE void frinti(FPRegisterID vd, FPRegisterID vn)
2387     {
2388         CHECK_DATASIZE();
2389         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FRINTI, vn, vd));
2390     }
2391
2392     template<int datasize>
2393     ALWAYS_INLINE void frintm(FPRegisterID vd, FPRegisterID vn)
2394     {
2395         CHECK_DATASIZE();
2396         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FRINTM, vn, vd));
2397     }
2398
2399     template<int datasize>
2400     ALWAYS_INLINE void frintn(FPRegisterID vd, FPRegisterID vn)
2401     {
2402         CHECK_DATASIZE();
2403         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FRINTN, vn, vd));
2404     }
2405
2406     template<int datasize>
2407     ALWAYS_INLINE void frintp(FPRegisterID vd, FPRegisterID vn)
2408     {
2409         CHECK_DATASIZE();
2410         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FRINTP, vn, vd));
2411     }
2412
2413     template<int datasize>
2414     ALWAYS_INLINE void frintx(FPRegisterID vd, FPRegisterID vn)
2415     {
2416         CHECK_DATASIZE();
2417         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FRINTX, vn, vd));
2418     }
2419
2420     template<int datasize>
2421     ALWAYS_INLINE void frintz(FPRegisterID vd, FPRegisterID vn)
2422     {
2423         CHECK_DATASIZE();
2424         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FRINTZ, vn, vd));
2425     }
2426
2427     template<int datasize>
2428     ALWAYS_INLINE void fsqrt(FPRegisterID vd, FPRegisterID vn)
2429     {
2430         CHECK_DATASIZE();
2431         insn(floatingPointDataProcessing1Source(DATASIZE, FPDataOp_FSQRT, vn, vd));
2432     }
2433
2434     template<int datasize>
2435     ALWAYS_INLINE void fsub(FPRegisterID vd, FPRegisterID vn, FPRegisterID vm)
2436     {
2437         CHECK_DATASIZE();
2438         insn(floatingPointDataProcessing2Source(DATASIZE, vm, FPDataOp_FSUB, vn, vd));
2439     }
2440
2441     template<int datasize>
2442     ALWAYS_INLINE void ldr(FPRegisterID rt, RegisterID rn, RegisterID rm)
2443     {
2444         ldr<datasize>(rt, rn, rm, UXTX, 0);
2445     }
2446
2447     template<int datasize>
2448     ALWAYS_INLINE void ldr(FPRegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
2449     {
2450         CHECK_FP_MEMOP_DATASIZE();
2451         insn(loadStoreRegisterRegisterOffset(MEMOPSIZE, true, datasize == 128 ? MemOp_LOAD_V128 : MemOp_LOAD, rm, extend, encodeShiftAmount<datasize>(amount), rn, rt));
2452     }
2453
2454     template<int datasize>
2455     ALWAYS_INLINE void ldr(FPRegisterID rt, RegisterID rn, unsigned pimm)
2456     {
2457         CHECK_FP_MEMOP_DATASIZE();
2458         insn(loadStoreRegisterUnsignedImmediate(MEMOPSIZE, true, datasize == 128 ? MemOp_LOAD_V128 : MemOp_LOAD, encodePositiveImmediate<datasize>(pimm), rn, rt));
2459     }
2460
2461     template<int datasize>
2462     ALWAYS_INLINE void ldr(FPRegisterID rt, RegisterID rn, PostIndex simm)
2463     {
2464         CHECK_FP_MEMOP_DATASIZE();
2465         insn(loadStoreRegisterPostIndex(MEMOPSIZE, true, datasize == 128 ? MemOp_LOAD_V128 : MemOp_LOAD, simm, rn, rt));
2466     }
2467
2468     template<int datasize>
2469     ALWAYS_INLINE void ldr(FPRegisterID rt, RegisterID rn, PreIndex simm)
2470     {
2471         CHECK_FP_MEMOP_DATASIZE();
2472         insn(loadStoreRegisterPreIndex(MEMOPSIZE, true, datasize == 128 ? MemOp_LOAD_V128 : MemOp_LOAD, simm, rn, rt));
2473     }
2474
2475     template<int datasize>
2476     ALWAYS_INLINE void ldr_literal(FPRegisterID rt, int offset = 0)
2477     {
2478         CHECK_FP_MEMOP_DATASIZE();
2479         ASSERT(datasize >= 32);
2480         ASSERT(!(offset & 3));
2481         insn(loadRegisterLiteral(datasize == 128 ? LdrLiteralOp_128BIT : datasize == 64 ? LdrLiteralOp_64BIT : LdrLiteralOp_32BIT, true, offset >> 2, rt));
2482     }
2483
2484     template<int datasize>
2485     ALWAYS_INLINE void ldur(FPRegisterID rt, RegisterID rn, int simm)
2486     {
2487         CHECK_FP_MEMOP_DATASIZE();
2488         insn(loadStoreRegisterUnscaledImmediate(MEMOPSIZE, true, datasize == 128 ? MemOp_LOAD_V128 : MemOp_LOAD, simm, rn, rt));
2489     }
2490
2491     template<int dstsize, int srcsize>
2492     ALWAYS_INLINE void scvtf(FPRegisterID vd, RegisterID rn)
2493     {
2494         CHECK_DATASIZE_OF(dstsize);
2495         CHECK_DATASIZE_OF(srcsize);
2496         insn(floatingPointIntegerConversions(DATASIZE_OF(srcsize), DATASIZE_OF(dstsize), FPIntConvOp_SCVTF, rn, vd));
2497     }
2498
2499     template<int datasize>
2500     ALWAYS_INLINE void str(FPRegisterID rt, RegisterID rn, RegisterID rm)
2501     {
2502         str<datasize>(rt, rn, rm, UXTX, 0);
2503     }
2504
2505     template<int datasize>
2506     ALWAYS_INLINE void str(FPRegisterID rt, RegisterID rn, RegisterID rm, ExtendType extend, int amount)
2507     {
2508         CHECK_FP_MEMOP_DATASIZE();
2509         insn(loadStoreRegisterRegisterOffset(MEMOPSIZE, true, datasize == 128 ? MemOp_STORE_V128 : MemOp_STORE, rm, extend, encodeShiftAmount<datasize>(amount), rn, rt));
2510     }
2511
2512     template<int datasize>
2513     ALWAYS_INLINE void str(FPRegisterID rt, RegisterID rn, unsigned pimm)
2514     {
2515         CHECK_FP_MEMOP_DATASIZE();
2516         insn(loadStoreRegisterUnsignedImmediate(MEMOPSIZE, true, datasize == 128 ? MemOp_STORE_V128 : MemOp_STORE, encodePositiveImmediate<datasize>(pimm), rn, rt));
2517     }
2518
2519     template<int datasize>
2520     ALWAYS_INLINE void str(FPRegisterID rt, RegisterID rn, PostIndex simm)
2521     {
2522         CHECK_FP_MEMOP_DATASIZE();
2523         insn(loadStoreRegisterPostIndex(MEMOPSIZE, true, datasize == 128 ? MemOp_STORE_V128 : MemOp_STORE, simm, rn, rt));
2524     }
2525
2526     template<int datasize>
2527     ALWAYS_INLINE void str(FPRegisterID rt, RegisterID rn, PreIndex simm)
2528     {
2529         CHECK_FP_MEMOP_DATASIZE();
2530         insn(loadStoreRegisterPreIndex(MEMOPSIZE, true, datasize == 128 ? MemOp_STORE_V128 : MemOp_STORE, simm, rn, rt));
2531     }
2532
2533     template<int datasize>
2534     ALWAYS_INLINE void stur(FPRegisterID rt, RegisterID rn, int simm)
2535     {
2536         CHECK_DATASIZE();
2537         insn(loadStoreRegisterUnscaledImmediate(MEMOPSIZE, true, datasize == 128 ? MemOp_STORE_V128 : MemOp_STORE, simm, rn, rt));
2538     }
2539
2540     template<int dstsize, int srcsize>
2541     ALWAYS_INLINE void ucvtf(FPRegisterID vd, RegisterID rn)
2542     {
2543         CHECK_DATASIZE_OF(dstsize);
2544         CHECK_DATASIZE_OF(srcsize);
2545         insn(floatingPointIntegerConversions(DATASIZE_OF(srcsize), DATASIZE_OF(dstsize), FPIntConvOp_UCVTF, rn, vd));
2546     }
2547
2548     // Admin methods:
2549
2550     AssemblerLabel labelIgnoringWatchpoints()
2551     {
2552         return m_buffer.label();
2553     }
2554
2555     AssemblerLabel labelForWatchpoint()
2556     {
2557         AssemblerLabel result = m_buffer.label();
2558         if (static_cast<int>(result.m_offset) != m_indexOfLastWatchpoint)
2559             result = label();
2560         m_indexOfLastWatchpoint = result.m_offset;
2561         m_indexOfTailOfLastWatchpoint = result.m_offset + maxJumpReplacementSize();
2562         return result;
2563     }
2564
2565     AssemblerLabel label()
2566     {
2567         AssemblerLabel result = m_buffer.label();
2568         while (UNLIKELY(static_cast<int>(result.m_offset) < m_indexOfTailOfLastWatchpoint)) {
2569             nop();
2570             result = m_buffer.label();
2571         }
2572         return result;
2573     }
2574
2575     AssemblerLabel align(int alignment)
2576     {
2577         ASSERT(!(alignment & 3));
2578         while (!m_buffer.isAligned(alignment))
2579             brk(0);
2580         return label();
2581     }
2582
2583     static void* getRelocatedAddress(void* code, AssemblerLabel label)
2584     {
2585         ASSERT(label.isSet());
2586         return reinterpret_cast<void*>(reinterpret_cast<ptrdiff_t>(code) + label.m_offset);
2587     }
2588
2589     static int getDifferenceBetweenLabels(AssemblerLabel a, AssemblerLabel b)
2590     {
2591         return b.m_offset - a.m_offset;
2592     }
2593
2594     void* unlinkedCode() { return m_buffer.data(); }
2595     size_t codeSize() const { return m_buffer.codeSize(); }
2596
2597     static unsigned getCallReturnOffset(AssemblerLabel call)
2598     {
2599         ASSERT(call.isSet());
2600         return call.m_offset;
2601     }
2602
2603     // Linking & patching:
2604     //
2605     // 'link' and 'patch' methods are for use on unprotected code - such as the code
2606     // within the AssemblerBuffer, and code being patched by the patch buffer. Once
2607     // code has been finalized it is (platform support permitting) within a non-
2608     // writable region of memory; to modify the code in an execute-only execuable
2609     // pool the 'repatch' and 'relink' methods should be used.
2610
2611     void linkJump(AssemblerLabel from, AssemblerLabel to, JumpType type, Condition condition)
2612     {
2613         ASSERT(to.isSet());
2614         ASSERT(from.isSet());
2615         m_jumpsToLink.append(LinkRecord(from.m_offset, to.m_offset, type, condition));
2616     }
2617
2618     void linkJump(AssemblerLabel from, AssemblerLabel to, JumpType type, Condition condition, bool is64Bit, RegisterID compareRegister)
2619     {
2620         ASSERT(to.isSet());
2621         ASSERT(from.isSet());
2622         m_jumpsToLink.append(LinkRecord(from.m_offset, to.m_offset, type, condition, is64Bit, compareRegister));
2623     }
2624
2625     void linkJump(AssemblerLabel from, AssemblerLabel to, JumpType type, Condition condition, unsigned bitNumber, RegisterID compareRegister)
2626     {
2627         ASSERT(to.isSet());
2628         ASSERT(from.isSet());
2629         m_jumpsToLink.append(LinkRecord(from.m_offset, to.m_offset, type, condition, bitNumber, compareRegister));
2630     }
2631
2632     void linkJump(AssemblerLabel from, AssemblerLabel to)
2633     {
2634         ASSERT(from.isSet());
2635         ASSERT(to.isSet());
2636         relinkJumpOrCall<false>(addressOf(from), addressOf(to));
2637     }
2638
2639     static void linkJump(void* code, AssemblerLabel from, void* to)
2640     {
2641         ASSERT(from.isSet());
2642         relinkJumpOrCall<false>(addressOf(code, from), to);
2643     }
2644
2645     static void linkCall(void* code, AssemblerLabel from, void* to)
2646     {
2647         ASSERT(from.isSet());
2648         linkJumpOrCall<true>(addressOf(code, from) - 1, to);
2649     }
2650
2651     static void linkPointer(void* code, AssemblerLabel where, void* valuePtr)
2652     {
2653         linkPointer(addressOf(code, where), valuePtr);
2654     }
2655
2656     static void replaceWithJump(void* where, void* to)
2657     {
2658         intptr_t offset = (reinterpret_cast<intptr_t>(to) - reinterpret_cast<intptr_t>(where)) >> 2;
2659         ASSERT(static_cast<int>(offset) == offset);
2660         *static_cast<int*>(where) = unconditionalBranchImmediate(false, static_cast<int>(offset));
2661         cacheFlush(where, sizeof(int));
2662     }
2663
2664     static ptrdiff_t maxJumpReplacementSize()
2665     {
2666         return 4;
2667     }
2668
2669     static void replaceWithLoad(void* where)
2670     {
2671         Datasize sf;
2672         AddOp op;
2673         SetFlags S;
2674         int shift;
2675         int imm12;
2676         RegisterID rn;
2677         RegisterID rd;
2678         if (disassembleAddSubtractImmediate(where, sf, op, S, shift, imm12, rn, rd)) {
2679             ASSERT(sf == Datasize_64);
2680             ASSERT(op == AddOp_ADD);
2681             ASSERT(!S);
2682             ASSERT(!shift);
2683             ASSERT(!(imm12 & ~0xff8));
2684             *static_cast<int*>(where) = loadStoreRegisterUnsignedImmediate(MemOpSize_64, false, MemOp_LOAD, encodePositiveImmediate<64>(imm12), rn, rd);
2685             cacheFlush(where, sizeof(int));
2686         }
2687 #if !ASSERT_DISABLED
2688         else {
2689             MemOpSize size;
2690             bool V;
2691             MemOp opc;
2692             int imm12;
2693             RegisterID rn;
2694             RegisterID rt;
2695             ASSERT(disassembleLoadStoreRegisterUnsignedImmediate(where, size, V, opc, imm12, rn, rt));
2696             ASSERT(size == MemOpSize_64);
2697             ASSERT(!V);
2698             ASSERT(opc == MemOp_LOAD);
2699             ASSERT(!(imm12 & ~0x1ff));
2700         }
2701 #endif
2702     }
2703
2704     static void replaceWithAddressComputation(void* where)
2705     {
2706         MemOpSize size;
2707         bool V;
2708         MemOp opc;
2709         int imm12;
2710         RegisterID rn;
2711         RegisterID rt;
2712         if (disassembleLoadStoreRegisterUnsignedImmediate(where, size, V, opc, imm12, rn, rt)) {
2713             ASSERT(size == MemOpSize_64);
2714             ASSERT(!V);
2715             ASSERT(opc == MemOp_LOAD);
2716             ASSERT(!(imm12 & ~0x1ff));
2717             *static_cast<int*>(where) = addSubtractImmediate(Datasize_64, AddOp_ADD, DontSetFlags, 0, imm12 * sizeof(void*), rn, rt);
2718             cacheFlush(where, sizeof(int));
2719         }
2720 #if !ASSERT_DISABLED
2721         else {
2722             Datasize sf;
2723             AddOp op;
2724             SetFlags S;
2725             int shift;
2726             int imm12;
2727             RegisterID rn;
2728             RegisterID rd;
2729             ASSERT(disassembleAddSubtractImmediate(where, sf, op, S, shift, imm12, rn, rd));
2730             ASSERT(sf == Datasize_64);
2731             ASSERT(op == AddOp_ADD);
2732             ASSERT(!S);
2733             ASSERT(!shift);
2734             ASSERT(!(imm12 & ~0xff8));
2735         }
2736 #endif
2737     }
2738
2739     static void repatchPointer(void* where, void* valuePtr)
2740     {
2741         linkPointer(static_cast<int*>(where), valuePtr, true);
2742     }
2743
2744     static void setPointer(int* address, void* valuePtr, RegisterID rd, bool flush)
2745     {
2746         uintptr_t value = reinterpret_cast<uintptr_t>(valuePtr);
2747         address[0] = moveWideImediate(Datasize_64, MoveWideOp_Z, 0, getHalfword(value, 0), rd);
2748         address[1] = moveWideImediate(Datasize_64, MoveWideOp_K, 1, getHalfword(value, 1), rd);
2749         address[2] = moveWideImediate(Datasize_64, MoveWideOp_K, 2, getHalfword(value, 2), rd);
2750
2751         if (flush)
2752             cacheFlush(address, sizeof(int) * 3);
2753     }
2754
2755     static void repatchInt32(void* where, int32_t value)
2756     {
2757         int* address = static_cast<int*>(where);
2758
2759         Datasize sf;
2760         MoveWideOp opc;
2761         int hw;
2762         uint16_t imm16;
2763         RegisterID rd;
2764         bool expected = disassembleMoveWideImediate(address, sf, opc, hw, imm16, rd);
2765         ASSERT_UNUSED(expected, expected && !sf && (opc == MoveWideOp_Z || opc == MoveWideOp_N) && !hw);
2766         ASSERT(checkMovk<Datasize_32>(address[1], 1, rd));
2767
2768         if (value >= 0) {
2769             address[0] = moveWideImediate(Datasize_32, MoveWideOp_Z, 0, getHalfword(value, 0), rd);
2770             address[1] = moveWideImediate(Datasize_32, MoveWideOp_K, 1, getHalfword(value, 1), rd);
2771         } else {
2772             address[0] = moveWideImediate(Datasize_32, MoveWideOp_N, 0, ~getHalfword(value, 0), rd);
2773             address[1] = moveWideImediate(Datasize_32, MoveWideOp_K, 1, getHalfword(value, 1), rd);
2774         }
2775
2776         cacheFlush(where, sizeof(int) * 2);
2777     }
2778
2779     static void* readPointer(void* where)
2780     {
2781         int* address = static_cast<int*>(where);
2782
2783         Datasize sf;
2784         MoveWideOp opc;
2785         int hw;
2786         uint16_t imm16;
2787         RegisterID rdFirst, rd;
2788
2789         bool expected = disassembleMoveWideImediate(address, sf, opc, hw, imm16, rdFirst);
2790         ASSERT_UNUSED(expected, expected && sf && opc == MoveWideOp_Z && !hw);
2791         uintptr_t result = imm16;
2792
2793         expected = disassembleMoveWideImediate(address + 1, sf, opc, hw, imm16, rd);
2794         ASSERT_UNUSED(expected, expected && sf && opc == MoveWideOp_K && hw == 1 && rd == rdFirst);
2795         result |= static_cast<uintptr_t>(imm16) << 16;
2796
2797         expected = disassembleMoveWideImediate(address + 2, sf, opc, hw, imm16, rd);
2798         ASSERT_UNUSED(expected, expected && sf && opc == MoveWideOp_K && hw == 2 && rd == rdFirst);
2799         result |= static_cast<uintptr_t>(imm16) << 32;
2800
2801         return reinterpret_cast<void*>(result);
2802     }
2803
2804     static void* readCallTarget(void* from)
2805     {
2806         return readPointer(reinterpret_cast<int*>(from) - 4);
2807     }
2808
2809     static void relinkJump(void* from, void* to)
2810     {
2811         relinkJumpOrCall<false>(reinterpret_cast<int*>(from), to);
2812         cacheFlush(from, sizeof(int));
2813     }
2814
2815     static void relinkCall(void* from, void* to)
2816     {
2817         relinkJumpOrCall<true>(reinterpret_cast<int*>(from) - 1, to);
2818         cacheFlush(reinterpret_cast<int*>(from) - 1, sizeof(int));
2819     }
2820
2821     static void repatchCompact(void* where, int32_t value)
2822     {
2823         ASSERT(!(value & ~0x3ff8));
2824
2825         MemOpSize size;
2826         bool V;
2827         MemOp opc;
2828         int imm12;
2829         RegisterID rn;
2830         RegisterID rt;
2831         bool expected = disassembleLoadStoreRegisterUnsignedImmediate(where, size, V, opc, imm12, rn, rt);
2832         ASSERT_UNUSED(expected, expected && size >= MemOpSize_32 && !V && opc == MemOp_LOAD); // expect 32/64 bit load to GPR.
2833
2834         if (size == MemOpSize_32)
2835             imm12 = encodePositiveImmediate<32>(value);
2836         else
2837             imm12 = encodePositiveImmediate<64>(value);
2838         *static_cast<int*>(where) = loadStoreRegisterUnsignedImmediate(size, V, opc, imm12, rn, rt);
2839
2840         cacheFlush(where, sizeof(int));
2841     }
2842
2843     unsigned debugOffset() { return m_buffer.debugOffset(); }
2844
2845 #if OS(LINUX) && COMPILER(GCC)
2846     static inline void linuxPageFlush(uintptr_t begin, uintptr_t end)
2847     {
2848         __builtin___clear_cache(reinterpret_cast<void*>(begin), reinterpret_cast<void*>(end));
2849     }
2850 #endif
2851
2852     static void cacheFlush(void* code, size_t size)
2853     {
2854 #if OS(IOS)
2855         sys_cache_control(kCacheFunctionPrepareForExecution, code, size);
2856 #elif OS(LINUX)
2857         size_t page = pageSize();
2858         uintptr_t current = reinterpret_cast<uintptr_t>(code);
2859         uintptr_t end = current + size;
2860         uintptr_t firstPageEnd = (current & ~(page - 1)) + page;
2861
2862         if (end <= firstPageEnd) {
2863             linuxPageFlush(current, end);
2864             return;
2865         }
2866
2867         linuxPageFlush(current, firstPageEnd);
2868
2869         for (current = firstPageEnd; current + page < end; current += page)
2870             linuxPageFlush(current, current + page);
2871
2872         linuxPageFlush(current, end);
2873 #else
2874 #error "The cacheFlush support is missing on this platform."
2875 #endif
2876     }
2877
2878     // Assembler admin methods:
2879
2880     static int jumpSizeDelta(JumpType jumpType, JumpLinkType jumpLinkType) { return JUMP_ENUM_SIZE(jumpType) - JUMP_ENUM_SIZE(jumpLinkType); }
2881
2882     static ALWAYS_INLINE bool linkRecordSourceComparator(const LinkRecord& a, const LinkRecord& b)
2883     {
2884         return a.from() < b.from();
2885     }
2886
2887     static bool canCompact(JumpType jumpType)
2888     {
2889         // Fixed jumps cannot be compacted
2890         return (jumpType == JumpNoCondition) || (jumpType == JumpCondition) || (jumpType == JumpCompareAndBranch) || (jumpType == JumpTestBit);
2891     }
2892
2893     static JumpLinkType computeJumpType(JumpType jumpType, const uint8_t* from, const uint8_t* to)
2894     {
2895         switch (jumpType) {
2896         case JumpFixed:
2897             return LinkInvalid;
2898         case JumpNoConditionFixedSize:
2899             return LinkJumpNoCondition;
2900         case JumpConditionFixedSize:
2901             return LinkJumpCondition;
2902         case JumpCompareAndBranchFixedSize:
2903             return LinkJumpCompareAndBranch;
2904         case JumpTestBitFixedSize:
2905             return LinkJumpTestBit;
2906         case JumpNoCondition:
2907             return LinkJumpNoCondition;
2908         case JumpCondition: {
2909             ASSERT(!(reinterpret_cast<intptr_t>(from) & 0x3));
2910             ASSERT(!(reinterpret_cast<intptr_t>(to) & 0x3));
2911             intptr_t relative = reinterpret_cast<intptr_t>(to) - (reinterpret_cast<intptr_t>(from));
2912
2913             if (((relative << 43) >> 43) == relative)
2914                 return LinkJumpConditionDirect;
2915
2916             return LinkJumpCondition;
2917             }
2918         case JumpCompareAndBranch:  {
2919             ASSERT(!(reinterpret_cast<intptr_t>(from) & 0x3));
2920             ASSERT(!(reinterpret_cast<intptr_t>(to) & 0x3));
2921             intptr_t relative = reinterpret_cast<intptr_t>(to) - (reinterpret_cast<intptr_t>(from));
2922
2923             if (((relative << 43) >> 43) == relative)
2924                 return LinkJumpCompareAndBranchDirect;
2925
2926             return LinkJumpCompareAndBranch;
2927         }
2928         case JumpTestBit:   {
2929             ASSERT(!(reinterpret_cast<intptr_t>(from) & 0x3));
2930             ASSERT(!(reinterpret_cast<intptr_t>(to) & 0x3));
2931             intptr_t relative = reinterpret_cast<intptr_t>(to) - (reinterpret_cast<intptr_t>(from));
2932
2933             if (((relative << 50) >> 50) == relative)
2934                 return LinkJumpTestBitDirect;
2935
2936             return LinkJumpTestBit;
2937         }
2938         default:
2939             ASSERT_NOT_REACHED();
2940         }
2941
2942         return LinkJumpNoCondition;
2943     }
2944
2945     static JumpLinkType computeJumpType(LinkRecord& record, const uint8_t* from, const uint8_t* to)
2946     {
2947         JumpLinkType linkType = computeJumpType(record.type(), from, to);
2948         record.setLinkType(linkType);
2949         return linkType;
2950     }
2951
2952     Vector<LinkRecord, 0, UnsafeVectorOverflow>& jumpsToLink()
2953     {
2954         std::sort(m_jumpsToLink.begin(), m_jumpsToLink.end(), linkRecordSourceComparator);
2955         return m_jumpsToLink;
2956     }
2957
2958     static void ALWAYS_INLINE link(LinkRecord& record, uint8_t* from, uint8_t* to)
2959     {
2960         switch (record.linkType()) {
2961         case LinkJumpNoCondition:
2962             linkJumpOrCall<false>(reinterpret_cast<int*>(from), to);
2963             break;
2964         case LinkJumpConditionDirect:
2965             linkConditionalBranch<true>(record.condition(), reinterpret_cast<int*>(from), to);
2966             break;
2967         case LinkJumpCondition:
2968             linkConditionalBranch<false>(record.condition(), reinterpret_cast<int*>(from) - 1, to);
2969             break;
2970         case LinkJumpCompareAndBranchDirect:
2971             linkCompareAndBranch<true>(record.condition(), record.is64Bit(), record.compareRegister(), reinterpret_cast<int*>(from), to);
2972             break;
2973         case LinkJumpCompareAndBranch:
2974             linkCompareAndBranch<false>(record.condition(), record.is64Bit(), record.compareRegister(), reinterpret_cast<int*>(from) - 1, to);
2975             break;
2976         case LinkJumpTestBitDirect:
2977             linkTestAndBranch<true>(record.condition(), record.bitNumber(), record.compareRegister(), reinterpret_cast<int*>(from), to);
2978             break;
2979         case LinkJumpTestBit:
2980             linkTestAndBranch<false>(record.condition(), record.bitNumber(), record.compareRegister(), reinterpret_cast<int*>(from) - 1, to);
2981             break;
2982         default:
2983             ASSERT_NOT_REACHED();
2984             break;
2985         }
2986     }
2987
2988 private:
2989     template<Datasize size>
2990     static bool checkMovk(int insn, int _hw, RegisterID _rd)
2991     {
2992         Datasize sf;
2993         MoveWideOp opc;
2994         int hw;
2995         uint16_t imm16;
2996         RegisterID rd;
2997         bool expected = disassembleMoveWideImediate(&insn, sf, opc, hw, imm16, rd);
2998
2999         return expected
3000             && sf == size
3001             && opc == MoveWideOp_K
3002             && hw == _hw
3003             && rd == _rd;
3004     }
3005
3006     static void linkPointer(int* address, void* valuePtr, bool flush = false)
3007     {
3008         Datasize sf;
3009         MoveWideOp opc;
3010         int hw;
3011         uint16_t imm16;
3012         RegisterID rd;
3013         bool expected = disassembleMoveWideImediate(address, sf, opc, hw, imm16, rd);
3014         ASSERT_UNUSED(expected, expected && sf && opc == MoveWideOp_Z && !hw);
3015         ASSERT(checkMovk<Datasize_64>(address[1], 1, rd));
3016         ASSERT(checkMovk<Datasize_64>(address[2], 2, rd));
3017
3018         setPointer(address, valuePtr, rd, flush);
3019     }
3020
3021     template<bool isCall>
3022     static void linkJumpOrCall(int* from, void* to)
3023     {
3024         bool link;
3025         int imm26;
3026         bool isUnconditionalBranchImmediateOrNop = disassembleUnconditionalBranchImmediate(from, link, imm26) || disassembleNop(from);
3027
3028         ASSERT_UNUSED(isUnconditionalBranchImmediateOrNop, isUnconditionalBranchImmediateOrNop);
3029         ASSERT_UNUSED(isCall, (link == isCall) || disassembleNop(from));
3030         ASSERT(!(reinterpret_cast<intptr_t>(from) & 3));
3031         ASSERT(!(reinterpret_cast<intptr_t>(to) & 3));
3032         intptr_t offset = (reinterpret_cast<intptr_t>(to) - reinterpret_cast<intptr_t>(from)) >> 2;
3033         ASSERT(static_cast<int>(offset) == offset);
3034
3035         *from = unconditionalBranchImmediate(isCall, static_cast<int>(offset));
3036     }
3037
3038     template<bool isDirect>
3039     static void linkCompareAndBranch(Condition condition, bool is64Bit, RegisterID rt, int* from, void* to)
3040     {
3041         ASSERT(!(reinterpret_cast<intptr_t>(from) & 3));
3042         ASSERT(!(reinterpret_cast<intptr_t>(to) & 3));
3043         intptr_t offset = (reinterpret_cast<intptr_t>(to) - reinterpret_cast<intptr_t>(from)) >> 2;
3044         ASSERT(((offset << 38) >> 38) == offset);
3045
3046         bool useDirect = ((offset << 45) >> 45) == offset; // Fits in 19 bits
3047         ASSERT(!isDirect || useDirect);
3048
3049         if (useDirect || isDirect) {
3050             *from = compareAndBranchImmediate(is64Bit ? Datasize_64 : Datasize_32, condition == ConditionNE, static_cast<int>(offset), rt);
3051             if (!isDirect)
3052                 *(from + 1) = nopPseudo();
3053         } else {
3054             *from = compareAndBranchImmediate(is64Bit ? Datasize_64 : Datasize_32, invert(condition) == ConditionNE, 2, rt);
3055             linkJumpOrCall<false>(from + 1, to);
3056         }
3057     }
3058
3059     template<bool isDirect>
3060     static void linkConditionalBranch(Condition condition, int* from, void* to)
3061     {
3062         ASSERT(!(reinterpret_cast<intptr_t>(from) & 3));
3063         ASSERT(!(reinterpret_cast<intptr_t>(to) & 3));
3064         intptr_t offset = (reinterpret_cast<intptr_t>(to) - reinterpret_cast<intptr_t>(from)) >> 2;
3065         ASSERT(((offset << 38) >> 38) == offset);
3066
3067         bool useDirect = ((offset << 45) >> 45) == offset; // Fits in 19 bits
3068         ASSERT(!isDirect || useDirect);
3069
3070         if (useDirect || isDirect) {
3071             *from = conditionalBranchImmediate(static_cast<int>(offset), condition);
3072             if (!isDirect)
3073                 *(from + 1) = nopPseudo();
3074         } else {
3075             *from = conditionalBranchImmediate(2, invert(condition));
3076             linkJumpOrCall<false>(from + 1, to);
3077         }
3078     }
3079
3080     template<bool isDirect>
3081     static void linkTestAndBranch(Condition condition, unsigned bitNumber, RegisterID rt, int* from, void* to)
3082     {
3083         ASSERT(!(reinterpret_cast<intptr_t>(from) & 3));
3084         ASSERT(!(reinterpret_cast<intptr_t>(to) & 3));
3085         intptr_t offset = (reinterpret_cast<intptr_t>(to) - reinterpret_cast<intptr_t>(from)) >> 2;
3086         ASSERT(static_cast<int>(offset) == offset);
3087         ASSERT(((offset << 38) >> 38) == offset);
3088
3089         bool useDirect = ((offset << 50) >> 50) == offset; // Fits in 14 bits
3090         ASSERT(!isDirect || useDirect);
3091
3092         if (useDirect || isDirect) {
3093             *from = testAndBranchImmediate(condition == ConditionNE, static_cast<int>(bitNumber), static_cast<int>(offset), rt);
3094             if (!isDirect)
3095                 *(from + 1) = nopPseudo();
3096         } else {
3097             *from = testAndBranchImmediate(invert(condition) == ConditionNE, static_cast<int>(bitNumber), 2, rt);
3098             linkJumpOrCall<false>(from + 1, to);
3099         }
3100     }
3101
3102     template<bool isCall>
3103     static void relinkJumpOrCall(int* from, void* to)
3104     {
3105         if (!isCall && disassembleNop(from)) {
3106             unsigned op01;
3107             int imm19;
3108             Condition condition;
3109             bool isConditionalBranchImmediate = disassembleConditionalBranchImmediate(from - 1, op01, imm19, condition);
3110
3111             if (isConditionalBranchImmediate) {
3112                 ASSERT_UNUSED(op01, !op01);
3113                 ASSERT_UNUSED(isCall, !isCall);
3114
3115                 if (imm19 == 8)
3116                     condition = invert(condition);
3117
3118                 linkConditionalBranch<false>(condition, from - 1, to);
3119                 return;
3120             }
3121
3122             Datasize opSize;
3123             bool op;
3124             RegisterID rt;
3125             bool isCompareAndBranchImmediate = disassembleCompareAndBranchImmediate(from - 1, opSize, op, imm19, rt);
3126
3127             if (isCompareAndBranchImmediate) {
3128                 if (imm19 == 8)
3129                     op = !op;
3130
3131                 linkCompareAndBranch<false>(op ? ConditionNE : ConditionEQ, opSize == Datasize_64, rt, from - 1, to);
3132                 return;
3133             }
3134
3135             int imm14;
3136             unsigned bitNumber;
3137             bool isTestAndBranchImmediate = disassembleTestAndBranchImmediate(from - 1, op, bitNumber, imm14, rt);
3138
3139             if (isTestAndBranchImmediate) {
3140                 if (imm14 == 8)
3141                     op = !op;
3142
3143                 linkTestAndBranch<false>(op ? ConditionNE : ConditionEQ, bitNumber, rt, from - 1, to);
3144                 return;
3145             }
3146         }
3147
3148         linkJumpOrCall<isCall>(from, to);
3149     }
3150
3151     static int* addressOf(void* code, AssemblerLabel label)
3152     {
3153         return reinterpret_cast<int*>(static_cast<char*>(code) + label.m_offset);
3154     }
3155
3156     int* addressOf(AssemblerLabel label)
3157     {
3158         return addressOf(m_buffer.data(), label);
3159     }
3160
3161     static RegisterID disassembleXOrSp(int reg) { return reg == 31 ? ARM64Registers::sp : static_cast<RegisterID>(reg); }
3162     static RegisterID disassembleXOrZr(int reg) { return reg == 31 ? ARM64Registers::zr : static_cast<RegisterID>(reg); }
3163     static RegisterID disassembleXOrZrOrSp(bool useZr, int reg) { return reg == 31 ? (useZr ? ARM64Registers::zr : ARM64Registers::sp) : static_cast<RegisterID>(reg); }
3164
3165     static bool disassembleAddSubtractImmediate(void* address, Datasize& sf, AddOp& op, SetFlags& S, int& shift, int& imm12, RegisterID& rn, RegisterID& rd)
3166     {
3167         int insn = *static_cast<int*>(address);
3168         sf = static_cast<Datasize>((insn >> 31) & 1);
3169         op = static_cast<AddOp>((insn >> 30) & 1);
3170         S = static_cast<SetFlags>((insn >> 29) & 1);
3171         shift = (insn >> 22) & 3;
3172         imm12 = (insn >> 10) & 0x3ff;
3173         rn = disassembleXOrSp((insn >> 5) & 0x1f);
3174         rd = disassembleXOrZrOrSp(S, insn & 0x1f);
3175         return (insn & 0x1f000000) == 0x11000000;
3176     }
3177
3178     static bool disassembleLoadStoreRegisterUnsignedImmediate(void* address, MemOpSize& size, bool& V, MemOp& opc, int& imm12, RegisterID& rn, RegisterID& rt)
3179     {
3180         int insn = *static_cast<int*>(address);
3181         size = static_cast<MemOpSize>((insn >> 30) & 3);
3182         V = (insn >> 26) & 1;
3183         opc = static_cast<MemOp>((insn >> 22) & 3);
3184         imm12 = (insn >> 10) & 0xfff;
3185         rn = disassembleXOrSp((insn >> 5) & 0x1f);
3186         rt = disassembleXOrZr(insn & 0x1f);
3187         return (insn & 0x3b000000) == 0x39000000;
3188     }
3189
3190     static bool disassembleMoveWideImediate(void* address, Datasize& sf, MoveWideOp& opc, int& hw, uint16_t& imm16, RegisterID& rd)
3191     {
3192         int insn = *static_cast<int*>(address);
3193         sf = static_cast<Datasize>((insn >> 31) & 1);
3194         opc = static_cast<MoveWideOp>((insn >> 29) & 3);
3195         hw = (insn >> 21) & 3;
3196         imm16 = insn >> 5;
3197         rd = disassembleXOrZr(insn & 0x1f);
3198         return (insn & 0x1f800000) == 0x12800000;
3199     }
3200
3201     static bool disassembleNop(void* address)
3202     {
3203         unsigned insn = *static_cast<unsigned*>(address);
3204         return insn == 0xd503201f;
3205     }
3206
3207     static bool disassembleCompareAndBranchImmediate(void* address, Datasize& sf, bool& op, int& imm19, RegisterID& rt)
3208     {
3209         int insn = *static_cast<int*>(address);
3210         sf = static_cast<Datasize>((insn >> 31) & 1);
3211         op = (insn >> 24) & 0x1;
3212         imm19 = (insn << 8) >> 13;
3213         rt = static_cast<RegisterID>(insn & 0x1f);
3214         return (insn & 0x7e000000) == 0x34000000;
3215
3216     }
3217
3218     static bool disassembleConditionalBranchImmediate(void* address, unsigned& op01, int& imm19, Condition &condition)
3219     {
3220         int insn = *static_cast<int*>(address);
3221         op01 = ((insn >> 23) & 0x2) | ((insn >> 4) & 0x1);
3222         imm19 = (insn << 8) >> 13;
3223         condition = static_cast<Condition>(insn & 0xf);
3224         return (insn & 0xfe000000) == 0x54000000;
3225     }
3226
3227     static bool disassembleTestAndBranchImmediate(void* address, bool& op, unsigned& bitNumber, int& imm14, RegisterID& rt)
3228     {
3229         int insn = *static_cast<int*>(address);
3230         op = (insn >> 24) & 0x1;
3231         imm14 = (insn << 13) >> 18;
3232         bitNumber = static_cast<unsigned>((((insn >> 26) & 0x20)) | ((insn > 19) & 0x1f));
3233         rt = static_cast<RegisterID>(insn & 0x1f);
3234         return (insn & 0x7e000000) == 0x36000000;
3235
3236     }
3237
3238     static bool disassembleUnconditionalBranchImmediate(void* address, bool& op, int& imm26)
3239     {
3240         int insn = *static_cast<int*>(address);
3241         op = (insn >> 31) & 1;
3242         imm26 = (insn << 6) >> 6;
3243         return (insn & 0x7c000000) == 0x14000000;
3244     }
3245
3246     static int xOrSp(RegisterID reg) { ASSERT(!isZr(reg)); return reg; }
3247     static int xOrZr(RegisterID reg) { ASSERT(!isSp(reg)); return reg & 31; }
3248     static FPRegisterID xOrZrAsFPR(RegisterID reg) { return static_cast<FPRegisterID>(xOrZr(reg)); }
3249     static int xOrZrOrSp(bool useZr, RegisterID reg) { return useZr ? xOrZr(reg) : xOrSp(reg); }
3250
3251     ALWAYS_INLINE void insn(int instruction)
3252     {
3253         m_buffer.putInt(instruction);
3254     }
3255
3256     ALWAYS_INLINE static int addSubtractExtendedRegister(Datasize sf, AddOp op, SetFlags S, RegisterID rm, ExtendType option, int imm3, RegisterID rn, RegisterID rd)
3257     {
3258         ASSERT(imm3 < 5);
3259         // The only allocated values for opt is 0.
3260         const int opt = 0;
3261         return (0x0b200000 | sf << 31 | op << 30 | S << 29 | opt << 22 | xOrZr(rm) << 16 | option << 13 | (imm3 & 0x7) << 10 | xOrSp(rn) << 5 | xOrZrOrSp(S, rd));
3262     }
3263
3264     ALWAYS_INLINE static int addSubtractImmediate(Datasize sf, AddOp op, SetFlags S, int shift, int imm12, RegisterID rn, RegisterID rd)
3265     {
3266         ASSERT(shift < 2);
3267         ASSERT(isUInt12(imm12));
3268         return (0x11000000 | sf << 31 | op << 30 | S << 29 | shift << 22 | (imm12 & 0xfff) << 10 | xOrSp(rn) << 5 | xOrZrOrSp(S, rd));
3269     }
3270
3271     ALWAYS_INLINE static int addSubtractShiftedRegister(Datasize sf, AddOp op, SetFlags S, ShiftType shift, RegisterID rm, int imm6, RegisterID rn, RegisterID rd)
3272     {
3273         ASSERT(shift < 3);
3274         ASSERT(!(imm6 & (sf ? ~63 : ~31)));
3275         return (0x0b000000 | sf << 31 | op << 30 | S << 29 | shift << 22 | xOrZr(rm) << 16 | (imm6 & 0x3f) << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3276     }
3277
3278     ALWAYS_INLINE static int addSubtractWithCarry(Datasize sf, AddOp op, SetFlags S, RegisterID rm, RegisterID rn, RegisterID rd)
3279     {
3280         const int opcode2 = 0;
3281         return (0x1a000000 | sf << 31 | op << 30 | S << 29 | xOrZr(rm) << 16 | opcode2 << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3282     }
3283
3284     ALWAYS_INLINE static int bitfield(Datasize sf, BitfieldOp opc, int immr, int imms, RegisterID rn, RegisterID rd)
3285     {
3286         ASSERT(immr < (sf ? 64 : 32));
3287         ASSERT(imms < (sf ? 64 : 32));
3288         const int N = sf;
3289         return (0x13000000 | sf << 31 | opc << 29 | N << 22 | immr << 16 | imms << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3290     }
3291
3292     // 'op' means negate
3293     ALWAYS_INLINE static int compareAndBranchImmediate(Datasize sf, bool op, int32_t imm19, RegisterID rt)
3294     {
3295         ASSERT(imm19 == (imm19 << 13) >> 13);
3296         return (0x34000000 | sf << 31 | op << 24 | (imm19 & 0x7ffff) << 5 | xOrZr(rt));
3297     }
3298
3299     ALWAYS_INLINE static int conditionalBranchImmediate(int32_t imm19, Condition cond)
3300     {
3301         ASSERT(imm19 == (imm19 << 13) >> 13);
3302         ASSERT(!(cond & ~15));
3303         // The only allocated values for o1 & o0 are 0.
3304         const int o1 = 0;
3305         const int o0 = 0;
3306         return (0x54000000 | o1 << 24 | (imm19 & 0x7ffff) << 5 | o0 << 4 | cond);
3307     }
3308
3309     ALWAYS_INLINE static int conditionalCompareImmediate(Datasize sf, AddOp op, int imm5, Condition cond, RegisterID rn, int nzcv)
3310     {
3311         ASSERT(!(imm5 & ~0x1f));
3312         ASSERT(nzcv < 16);
3313         const int S = 1;
3314         const int o2 = 0;
3315         const int o3 = 0;
3316         return (0x1a400800 | sf << 31 | op << 30 | S << 29 | (imm5 & 0x1f) << 16 | cond << 12 | o2 << 10 | xOrZr(rn) << 5 | o3 << 4 | nzcv);
3317     }
3318
3319     ALWAYS_INLINE static int conditionalCompareRegister(Datasize sf, AddOp op, RegisterID rm, Condition cond, RegisterID rn, int nzcv)
3320     {
3321         ASSERT(nzcv < 16);
3322         const int S = 1;
3323         const int o2 = 0;
3324         const int o3 = 0;
3325         return (0x1a400000 | sf << 31 | op << 30 | S << 29 | xOrZr(rm) << 16 | cond << 12 | o2 << 10 | xOrZr(rn) << 5 | o3 << 4 | nzcv);
3326     }
3327
3328     // 'op' means negate
3329     // 'op2' means increment
3330     ALWAYS_INLINE static int conditionalSelect(Datasize sf, bool op, RegisterID rm, Condition cond, bool op2, RegisterID rn, RegisterID rd)
3331     {
3332         const int S = 0;
3333         return (0x1a800000 | sf << 31 | op << 30 | S << 29 | xOrZr(rm) << 16 | cond << 12 | op2 << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3334     }
3335
3336     ALWAYS_INLINE static int dataProcessing1Source(Datasize sf, DataOp1Source opcode, RegisterID rn, RegisterID rd)
3337     {
3338         const int S = 0;
3339         const int opcode2 = 0;
3340         return (0x5ac00000 | sf << 31 | S << 29 | opcode2 << 16 | opcode << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3341     }
3342
3343     ALWAYS_INLINE static int dataProcessing2Source(Datasize sf, RegisterID rm, DataOp2Source opcode, RegisterID rn, RegisterID rd)
3344     {
3345         const int S = 0;
3346         return (0x1ac00000 | sf << 31 | S << 29 | xOrZr(rm) << 16 | opcode << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3347     }
3348
3349     ALWAYS_INLINE static int dataProcessing3Source(Datasize sf, DataOp3Source opcode, RegisterID rm, RegisterID ra, RegisterID rn, RegisterID rd)
3350     {
3351         int op54 = opcode >> 4;
3352         int op31 = (opcode >> 1) & 7;
3353         int op0 = opcode & 1;
3354         return (0x1b000000 | sf << 31 | op54 << 29 | op31 << 21 | xOrZr(rm) << 16 | op0 << 15 | xOrZr(ra) << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3355     }
3356
3357     ALWAYS_INLINE static int excepnGeneration(ExcepnOp opc, uint16_t imm16, int LL)
3358     {
3359         ASSERT((opc == ExcepnOp_BREAKPOINT || opc == ExcepnOp_HALT) ? !LL : (LL && (LL < 4)));
3360         const int op2 = 0;
3361         return (0xd4000000 | opc << 21 | imm16 << 5 | op2 << 2 | LL);
3362     }
3363
3364     ALWAYS_INLINE static int extract(Datasize sf, RegisterID rm, int imms, RegisterID rn, RegisterID rd)
3365     {
3366         ASSERT(imms < (sf ? 64 : 32));
3367         const int op21 = 0;
3368         const int N = sf;
3369         const int o0 = 0;
3370         return (0x13800000 | sf << 31 | op21 << 29 | N << 22 | o0 << 21 | xOrZr(rm) << 16 | imms << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3371     }
3372
3373     ALWAYS_INLINE static int floatingPointCompare(Datasize type, FPRegisterID rm, FPRegisterID rn, FPCmpOp opcode2)
3374     {
3375         const int M = 0;
3376         const int S = 0;
3377         const int op = 0;
3378         return (0x1e202000 | M << 31 | S << 29 | type << 22 | rm << 16 | op << 14 | rn << 5 | opcode2);
3379     }
3380
3381     ALWAYS_INLINE static int floatingPointConditionalCompare(Datasize type, FPRegisterID rm, Condition cond, FPRegisterID rn, FPCondCmpOp op, int nzcv)
3382     {
3383         ASSERT(nzcv < 16);
3384         const int M = 0;
3385         const int S = 0;
3386         return (0x1e200400 | M << 31 | S << 29 | type << 22 | rm << 16 | cond << 12 | rn << 5 | op << 4 | nzcv);
3387     }
3388
3389     ALWAYS_INLINE static int floatingPointConditionalSelect(Datasize type, FPRegisterID rm, Condition cond, FPRegisterID rn, FPRegisterID rd)
3390     {
3391         const int M = 0;
3392         const int S = 0;
3393         return (0x1e200c00 | M << 31 | S << 29 | type << 22 | rm << 16 | cond << 12 | rn << 5 | rd);
3394     }
3395
3396     ALWAYS_INLINE static int floatingPointImmediate(Datasize type, int imm8, FPRegisterID rd)
3397     {
3398         const int M = 0;
3399         const int S = 0;
3400         const int imm5 = 0;
3401         return (0x1e201000 | M << 31 | S << 29 | type << 22 | (imm8 & 0xff) << 13 | imm5 << 5 | rd);
3402     }
3403
3404     ALWAYS_INLINE static int floatingPointIntegerConversions(Datasize sf, Datasize type, FPIntConvOp rmodeOpcode, FPRegisterID rn, FPRegisterID rd)
3405     {
3406         const int S = 0;
3407         return (0x1e200000 | sf << 31 | S << 29 | type << 22 | rmodeOpcode << 16 | rn << 5 | rd);
3408     }
3409
3410     ALWAYS_INLINE static int floatingPointIntegerConversions(Datasize sf, Datasize type, FPIntConvOp rmodeOpcode, FPRegisterID rn, RegisterID rd)
3411     {
3412         return floatingPointIntegerConversions(sf, type, rmodeOpcode, rn, xOrZrAsFPR(rd));
3413     }
3414
3415     ALWAYS_INLINE static int floatingPointIntegerConversions(Datasize sf, Datasize type, FPIntConvOp rmodeOpcode, RegisterID rn, FPRegisterID rd)
3416     {
3417         return floatingPointIntegerConversions(sf, type, rmodeOpcode, xOrZrAsFPR(rn), rd);
3418     }
3419
3420     ALWAYS_INLINE static int floatingPointDataProcessing1Source(Datasize type, FPDataOp1Source opcode, FPRegisterID rn, FPRegisterID rd)
3421     {
3422         const int M = 0;
3423         const int S = 0;
3424         return (0x1e204000 | M << 31 | S << 29 | type << 22 | opcode << 15 | rn << 5 | rd);
3425     }
3426
3427     ALWAYS_INLINE static int floatingPointDataProcessing2Source(Datasize type, FPRegisterID rm, FPDataOp2Source opcode, FPRegisterID rn, FPRegisterID rd)
3428     {
3429         const int M = 0;
3430         const int S = 0;
3431         return (0x1e200800 | M << 31 | S << 29 | type << 22 | rm << 16 | opcode << 12 | rn << 5 | rd);
3432     }
3433
3434     // 'o1' means negate
3435     ALWAYS_INLINE static int floatingPointDataProcessing3Source(Datasize type, bool o1, FPRegisterID rm, AddOp o2, FPRegisterID ra, FPRegisterID rn, FPRegisterID rd)
3436     {
3437         const int M = 0;
3438         const int S = 0;
3439         return (0x1f000000 | M << 31 | S << 29 | type << 22 | o1 << 21 | rm << 16 | o2 << 15 | ra << 10 | rn << 5 | rd);
3440     }
3441
3442     // 'V' means vector
3443     ALWAYS_INLINE static int loadRegisterLiteral(LdrLiteralOp opc, bool V, int imm19, FPRegisterID rt)
3444     {
3445         ASSERT(((imm19 << 13) >> 13) == imm19);
3446         return (0x18000000 | opc << 30 | V << 26 | (imm19 & 0x7ffff) << 5 | rt);
3447     }
3448
3449     ALWAYS_INLINE static int loadRegisterLiteral(LdrLiteralOp opc, bool V, int imm19, RegisterID rt)
3450     {
3451         return loadRegisterLiteral(opc, V, imm19, xOrZrAsFPR(rt));
3452     }
3453
3454     // 'V' means vector
3455     ALWAYS_INLINE static int loadStoreRegisterPostIndex(MemOpSize size, bool V, MemOp opc, int imm9, RegisterID rn, FPRegisterID rt)
3456     {
3457         ASSERT(!(size && V && (opc & 2))); // Maximum vector size is 128 bits.
3458         ASSERT(!((size & 2) && !V && (opc == 3))); // signed 32-bit load must be extending from 8/16 bits.
3459         ASSERT(isInt9(imm9));
3460         return (0x38000400 | size << 30 | V << 26 | opc << 22 | (imm9 & 0x1ff) << 12 | xOrSp(rn) << 5 | rt);
3461     }
3462
3463     ALWAYS_INLINE static int loadStoreRegisterPostIndex(MemOpSize size, bool V, MemOp opc, int imm9, RegisterID rn, RegisterID rt)
3464     {
3465         return loadStoreRegisterPostIndex(size, V, opc, imm9, rn, xOrZrAsFPR(rt));
3466     }
3467
3468     // 'V' means vector
3469     ALWAYS_INLINE static int loadStoreRegisterPairPostIndex(MemPairOpSize size, bool V, MemOp opc, int immediate, RegisterID rn, FPRegisterID rt, FPRegisterID rt2)
3470     {
3471         ASSERT(size < 3);
3472         ASSERT(opc == (opc & 1)); // Only load or store, load signed 64 is handled via size.
3473         ASSERT(V || (size != MemPairOp_LoadSigned_32) || (opc == MemOp_LOAD)); // There isn't an integer store signed.
3474         unsigned immedShiftAmount = memPairOffsetShift(V, size);
3475         int imm7 = immediate >> immedShiftAmount;
3476         ASSERT((imm7 << immedShiftAmount) == immediate && isInt7(imm7));
3477         return (0x28800000 | size << 30 | V << 26 | opc << 22 | (imm7 & 0x7f) << 15 | rt2 << 10 | xOrSp(rn) << 5 | rt);
3478     }
3479
3480     ALWAYS_INLINE static int loadStoreRegisterPairPostIndex(MemPairOpSize size, bool V, MemOp opc, int immediate, RegisterID rn, RegisterID rt, RegisterID rt2)
3481     {
3482         return loadStoreRegisterPairPostIndex(size, V, opc, immediate, rn, xOrZrAsFPR(rt), xOrZrAsFPR(rt2));
3483     }
3484
3485     // 'V' means vector
3486     ALWAYS_INLINE static int loadStoreRegisterPreIndex(MemOpSize size, bool V, MemOp opc, int imm9, RegisterID rn, FPRegisterID rt)
3487     {
3488         ASSERT(!(size && V && (opc & 2))); // Maximum vector size is 128 bits.
3489         ASSERT(!((size & 2) && !V && (opc == 3))); // signed 32-bit load must be extending from 8/16 bits.
3490         ASSERT(isInt9(imm9));
3491         return (0x38000c00 | size << 30 | V << 26 | opc << 22 | (imm9 & 0x1ff) << 12 | xOrSp(rn) << 5 | rt);
3492     }
3493
3494     ALWAYS_INLINE static int loadStoreRegisterPreIndex(MemOpSize size, bool V, MemOp opc, int imm9, RegisterID rn, RegisterID rt)
3495     {
3496         return loadStoreRegisterPreIndex(size, V, opc, imm9, rn, xOrZrAsFPR(rt));
3497     }
3498
3499     // 'V' means vector
3500     ALWAYS_INLINE static int loadStoreRegisterPairPreIndex(MemPairOpSize size, bool V, MemOp opc, int immediate, RegisterID rn, FPRegisterID rt, FPRegisterID rt2)
3501     {
3502         ASSERT(size < 3);
3503         ASSERT(opc == (opc & 1)); // Only load or store, load signed 64 is handled via size.
3504         ASSERT(V || (size != MemPairOp_LoadSigned_32) || (opc == MemOp_LOAD)); // There isn't an integer store signed.
3505         unsigned immedShiftAmount = memPairOffsetShift(V, size);
3506         int imm7 = immediate >> immedShiftAmount;
3507         ASSERT((imm7 << immedShiftAmount) == immediate && isInt7(imm7));
3508         return (0x29800000 | size << 30 | V << 26 | opc << 22 | (imm7 & 0x7f) << 15 | rt2 << 10 | xOrSp(rn) << 5 | rt);
3509     }
3510
3511     ALWAYS_INLINE static int loadStoreRegisterPairPreIndex(MemPairOpSize size, bool V, MemOp opc, int immediate, RegisterID rn, RegisterID rt, RegisterID rt2)
3512     {
3513         return loadStoreRegisterPairPreIndex(size, V, opc, immediate, rn, xOrZrAsFPR(rt), xOrZrAsFPR(rt2));
3514     }
3515
3516     // 'V' means vector
3517     // 'S' means shift rm
3518     ALWAYS_INLINE static int loadStoreRegisterRegisterOffset(MemOpSize size, bool V, MemOp opc, RegisterID rm, ExtendType option, bool S, RegisterID rn, FPRegisterID rt)
3519     {
3520         ASSERT(!(size && V && (opc & 2))); // Maximum vector size is 128 bits.
3521         ASSERT(!((size & 2) && !V && (opc == 3))); // signed 32-bit load must be extending from 8/16 bits.
3522         ASSERT(option & 2); // The ExtendType for the address must be 32/64 bit, signed or unsigned - not 8/16bit.
3523         return (0x38200800 | size << 30 | V << 26 | opc << 22 | xOrZr(rm) << 16 | option << 13 | S << 12 | xOrSp(rn) << 5 | rt);
3524     }
3525
3526     ALWAYS_INLINE static int loadStoreRegisterRegisterOffset(MemOpSize size, bool V, MemOp opc, RegisterID rm, ExtendType option, bool S, RegisterID rn, RegisterID rt)
3527     {
3528         return loadStoreRegisterRegisterOffset(size, V, opc, rm, option, S, rn, xOrZrAsFPR(rt));
3529     }
3530
3531     // 'V' means vector
3532     ALWAYS_INLINE static int loadStoreRegisterUnscaledImmediate(MemOpSize size, bool V, MemOp opc, int imm9, RegisterID rn, FPRegisterID rt)
3533     {
3534         ASSERT(!(size && V && (opc & 2))); // Maximum vector size is 128 bits.
3535         ASSERT(!((size & 2) && !V && (opc == 3))); // signed 32-bit load must be extending from 8/16 bits.
3536         ASSERT(isInt9(imm9));
3537         return (0x38000000 | size << 30 | V << 26 | opc << 22 | (imm9 & 0x1ff) << 12 | xOrSp(rn) << 5 | rt);
3538     }
3539
3540     ALWAYS_INLINE static int loadStoreRegisterUnscaledImmediate(MemOpSize size, bool V, MemOp opc, int imm9, RegisterID rn, RegisterID rt)
3541     {
3542         ASSERT(isInt9(imm9));
3543         return loadStoreRegisterUnscaledImmediate(size, V, opc, imm9, rn, xOrZrAsFPR(rt));
3544     }
3545
3546     // 'V' means vector
3547     ALWAYS_INLINE static int loadStoreRegisterUnsignedImmediate(MemOpSize size, bool V, MemOp opc, int imm12, RegisterID rn, FPRegisterID rt)
3548     {
3549         ASSERT(!(size && V && (opc & 2))); // Maximum vector size is 128 bits.
3550         ASSERT(!((size & 2) && !V && (opc == 3))); // signed 32-bit load must be extending from 8/16 bits.
3551         ASSERT(isUInt12(imm12));
3552         return (0x39000000 | size << 30 | V << 26 | opc << 22 | (imm12 & 0xfff) << 10 | xOrSp(rn) << 5 | rt);
3553     }
3554
3555     ALWAYS_INLINE static int loadStoreRegisterUnsignedImmediate(MemOpSize size, bool V, MemOp opc, int imm12, RegisterID rn, RegisterID rt)
3556     {
3557         return loadStoreRegisterUnsignedImmediate(size, V, opc, imm12, rn, xOrZrAsFPR(rt));
3558     }
3559
3560     ALWAYS_INLINE static int logicalImmediate(Datasize sf, LogicalOp opc, int N_immr_imms, RegisterID rn, RegisterID rd)
3561     {
3562         ASSERT(!(N_immr_imms & (sf ? ~0x1fff : ~0xfff)));
3563         return (0x12000000 | sf << 31 | opc << 29 | N_immr_imms << 10 | xOrZr(rn) << 5 | xOrZrOrSp(opc == LogicalOp_ANDS, rd));
3564     }
3565
3566     // 'N' means negate rm
3567     ALWAYS_INLINE static int logicalShiftedRegister(Datasize sf, LogicalOp opc, ShiftType shift, bool N, RegisterID rm, int imm6, RegisterID rn, RegisterID rd)
3568     {
3569         ASSERT(!(imm6 & (sf ? ~63 : ~31)));
3570         return (0x0a000000 | sf << 31 | opc << 29 | shift << 22 | N << 21 | xOrZr(rm) << 16 | (imm6 & 0x3f) << 10 | xOrZr(rn) << 5 | xOrZr(rd));
3571     }
3572
3573     ALWAYS_INLINE static int moveWideImediate(Datasize sf, MoveWideOp opc, int hw, uint16_t imm16, RegisterID rd)
3574     {
3575         ASSERT(hw < (sf ? 4 : 2));
3576         return (0x12800000 | sf << 31 | opc << 29 | hw << 21 | (int)imm16 << 5 | xOrZr(rd));
3577     }
3578
3579     // 'op' means link
3580     ALWAYS_INLINE static int unconditionalBranchImmediate(bool op, int32_t imm26)
3581     {
3582         ASSERT(imm26 == (imm26 << 6) >> 6);
3583         return (0x14000000 | op << 31 | (imm26 & 0x3ffffff));
3584     }
3585
3586     // 'op' means page
3587     ALWAYS_INLINE static int pcRelative(bool op, int32_t imm21, RegisterID rd)
3588     {
3589         ASSERT(imm21 == (imm21 << 11) >> 11);
3590         int32_t immlo = imm21 & 3;
3591         int32_t immhi = (imm21 >> 2) & 0x7ffff;
3592         return (0x10000000 | op << 31 | immlo << 29 | immhi << 5 | xOrZr(rd));
3593     }
3594
3595     ALWAYS_INLINE static int system(bool L, int op0, int op1, int crn, int crm, int op2, RegisterID rt)
3596     {
3597         return (0xd5000000 | L << 21 | op0 << 19 | op1 << 16 | crn << 12 | crm << 8 | op2 << 5 | xOrZr(rt));
3598     }
3599
3600     ALWAYS_INLINE static int hintPseudo(int imm)
3601     {
3602         ASSERT(!(imm & ~0x7f));
3603         return system(0, 0, 3, 2, (imm >> 3) & 0xf, imm & 0x7, ARM64Registers::zr);
3604     }
3605
3606     ALWAYS_INLINE static int nopPseudo()
3607     {
3608         return hintPseudo(0);
3609     }
3610
3611     // 'op' means negate
3612     ALWAYS_INLINE static int testAndBranchImmediate(bool op, int b50, int imm14, RegisterID rt)
3613     {
3614         ASSERT(!(b50 & ~0x3f));
3615         ASSERT(imm14 == (imm14 << 18) >> 18);
3616         int b5 = b50 >> 5;
3617         int b40 = b50 & 0x1f;
3618         return (0x36000000 | b5 << 31 | op << 24 | b40 << 19 | (imm14 & 0x3fff) << 5 | xOrZr(rt));
3619     }
3620
3621     ALWAYS_INLINE static int unconditionalBranchRegister(BranchType opc, RegisterID rn)
3622     {
3623         // The only allocated values for op2 is 0x1f, for op3 & op4 are 0.
3624         const int op2 = 0x1f;
3625         const int op3 = 0;
3626         const int op4 = 0;
3627         return (0xd6000000 | opc << 21 | op2 << 16 | op3 << 10 | xOrZr(rn) << 5 | op4);
3628     }
3629
3630     AssemblerBuffer m_buffer;
3631     Vector<LinkRecord, 0, UnsafeVectorOverflow> m_jumpsToLink;
3632     int m_indexOfLastWatchpoint;
3633     int m_indexOfTailOfLastWatchpoint;
3634 };
3635
3636 } // namespace JSC
3637
3638 #undef CHECK_DATASIZE_OF
3639 #undef DATASIZE_OF
3640 #undef MEMOPSIZE_OF
3641 #undef CHECK_DATASIZE
3642 #undef DATASIZE
3643 #undef MEMOPSIZE
3644 #undef CHECK_FP_MEMOP_DATASIZE
3645
3646 #endif // ENABLE(ASSEMBLER) && CPU(ARM64)
3647
3648 #endif // ARM64Assembler_h