llint/LowLevelInterpreter64.asm

   1 # Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions
   5 # are met:
   6 # 1. Redistributions of source code must retain the above copyright
   7 #    notice, this list of conditions and the following disclaimer.
   8 # 2. Redistributions in binary form must reproduce the above copyright
   9 #    notice, this list of conditions and the following disclaimer in the
  10 #    documentation and/or other materials provided with the distribution.
  11 #
  12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  13 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  14 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  15 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  16 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  17 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  18 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  19 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  20 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  21 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  22 # THE POSSIBILITY OF SUCH DAMAGE.
  23
  24
  25 # Utilities.
  26 macro jumpToInstruction()
  27     jmp [PB, PC, 8]
  28 end
  29
  30 macro dispatch(advance)
  31     addp advance, PC
  32     jumpToInstruction()
  33 end
  34
  35 macro dispatchInt(advance)
  36     addi advance, PC
  37     jumpToInstruction()
  38 end
  39
  40 macro dispatchIntIndirect(offset)
  41     dispatchInt(offset * 8[PB, PC, 8])
  42 end
  43
  44 macro dispatchAfterCall()
  45     loadi ArgumentCount + TagOffset[cfr], PC
  46     loadp CodeBlock[cfr], PB
  47     loadp CodeBlock::m_instructions[PB], PB
  48     loadisFromInstruction(1, t1)
  49     storeq t0, [cfr, t1, 8]
  50     valueProfile(t0, (CallOpCodeSize - 1), t2)
  51     dispatch(CallOpCodeSize)
  52 end
  53
  54 macro cCall2(function, arg1, arg2)
  55     checkStackPointerAlignment(t4, 0xbad0c002)
  56     if X86_64
  57         move arg1, t4
  58         move arg2, t5
  59         call function
  60     elsif X86_64_WIN
  61         # Note: this implementation is only correct if the return type size is > 8 bytes.
  62         # See macro cCall2Void for an implementation when the return type <= 8 bytes.
  63         # On Win64, when the return type is larger than 8 bytes, we need to allocate space on the stack for the return value.
  64         # On entry rcx (t2), should contain a pointer to this stack space. The other parameters are shifted to the right,
  65         # rdx (t1) should contain the first argument, and r8 (t6) should contain the second argument.
  66         # On return, rax contains a pointer to this stack value, and we then need to copy the 16 byte return value into rax (t0) and rdx (t1)
  67         # since the return value is expected to be split between the two.
  68         # See http://msdn.microsoft.com/en-us/library/7572ztz4.aspx
  69         move arg1, t1
  70         move arg2, t6
  71         subp 48, sp
  72         move sp, t2
  73         addp 32, t2
  74         call function
  75         addp 48, sp
  76         move 8[t0], t1
  77         move [t0], t0
  78     elsif ARM64
  79         move arg1, t0
  80         move arg2, t1
  81         call function
  82     elsif C_LOOP
  83         cloopCallSlowPath function, arg1, arg2
  84     else
  85         error
  86     end
  87 end
  88
  89 macro cCall2Void(function, arg1, arg2)
  90     if C_LOOP
  91         cloopCallSlowPathVoid function, arg1, arg2
  92     elsif X86_64_WIN
  93         # Note: we cannot use the cCall2 macro for Win64 in this case,
  94         # as the Win64 cCall2 implemenation is only correct when the return type size is > 8 bytes.
  95         # On Win64, rcx and rdx are used for passing the first two parameters.
  96         # We also need to make room on the stack for all four parameter registers.
  97         # See http://msdn.microsoft.com/en-us/library/ms235286.aspx
  98         move arg2, t1
  99         move arg1, t2
 100         subp 32, sp
 101         call function
 102         addp 32, sp
 103     else
 104         cCall2(function, arg1, arg2)
 105     end
 106 end
 107
 108 # This barely works. arg3 and arg4 should probably be immediates.
 109 macro cCall4(function, arg1, arg2, arg3, arg4)
 110     checkStackPointerAlignment(t4, 0xbad0c004)
 111     if X86_64
 112         move arg1, t4
 113         move arg2, t5
 114         move arg3, t1
 115         move arg4, t2
 116         call function
 117     elsif X86_64_WIN
 118         # On Win64, rcx, rdx, r8, and r9 are used for passing the first four parameters.
 119         # We also need to make room on the stack for all four parameter registers.
 120         # See http://msdn.microsoft.com/en-us/library/ms235286.aspx
 121         move arg1, t2
 122         move arg2, t1
 123         move arg3, t6
 124         move arg4, t7
 125         subp 32, sp
 126         call function
 127         addp 32, sp
 128     elsif ARM64
 129         move arg1, t0
 130         move arg2, t1
 131         move arg3, t2
 132         move arg4, t3
 133         call function
 134     elsif C_LOOP
 135         error
 136     else
 137         error
 138     end
 139 end
 140
 141 macro doCallToJavaScript(makeCall)
 142     if X86_64
 143         const entry = t4
 144         const vm = t5
 145         const protoCallFrame = t1
 146
 147         const previousCFR = t0
 148         const previousPC = t6
 149         const temp1 = t0
 150         const temp2 = t3
 151         const temp3 = t6
 152     elsif X86_64_WIN
 153         const entry = t2
 154         const vm = t1
 155         const protoCallFrame = t6
 156
 157         const previousCFR = t0
 158         const previousPC = t4
 159         const temp1 = t0
 160         const temp2 = t3
 161         const temp3 = t7
 162     elsif ARM64 or C_LOOP
 163         const entry = a0
 164         const vm = a1
 165         const protoCallFrame = a2
 166
 167         const previousCFR = t5
 168         const previousPC = lr
 169         const temp1 = t3
 170         const temp2 = t4
 171         const temp3 = t6
 172     end
 173
 174     callToJavaScriptPrologue()
 175
 176     if X86_64
 177         loadp 7*8[sp], previousPC
 178         move 6*8[sp], previousCFR
 179     elsif X86_64_WIN
 180         # Win64 pushes two more registers
 181         loadp 9*8[sp], previousPC
 182         move 8*8[sp], previousCFR
 183     elsif ARM64
 184         move cfr, previousCFR
 185     end
 186
 187     checkStackPointerAlignment(temp2, 0xbad0dc01)
 188
 189     # The stack reserved zone ensures that we have adequate space for the
 190     # VMEntrySentinelFrame. Proceed with allocating and initializing the
 191     # sentinel frame.
 192     move sp, cfr
 193     subp CallFrameHeaderSlots * 8, cfr
 194     storep 0, ArgumentCount[cfr]
 195     storep vm, Callee[cfr]
 196     loadp VM::topCallFrame[vm], temp2
 197     storep temp2, ScopeChain[cfr]
 198     storep 1, CodeBlock[cfr]
 199
 200     storep previousPC, ReturnPC[cfr]
 201     storep previousCFR, CallerFrame[cfr]
 202
 203     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], temp2
 204     addp CallFrameHeaderSlots, temp2, temp2
 205     lshiftp 3, temp2
 206     subp cfr, temp2, temp1
 207
 208     # Ensure that we have enough additional stack capacity for the incoming args,
 209     # and the frame for the JS code we're executing. We need to do this check
 210     # before we start copying the args from the protoCallFrame below.
 211     bpaeq temp1, VM::m_jsStackLimit[vm], .stackHeightOK
 212
 213     move cfr, sp
 214
 215     if C_LOOP
 216         move entry, temp2
 217         move vm, temp3
 218         cloopCallSlowPath _llint_stack_check_at_vm_entry, vm, temp1
 219         bpeq t0, 0, .stackCheckFailed
 220         move temp2, entry
 221         move temp3, vm
 222         jmp .stackHeightOK
 223
 224 .stackCheckFailed:
 225         move temp2, entry
 226         move temp3, vm
 227     end
 228
 229     cCall2(_llint_throw_stack_overflow_error, vm, protoCallFrame)
 230     callToJavaScriptEpilogue()
 231     ret
 232
 233 .stackHeightOK:
 234     move temp1, sp
 235     move 5, temp1
 236
 237 .copyHeaderLoop:
 238     subi 1, temp1
 239     loadq [protoCallFrame, temp1, 8], temp3
 240     storeq temp3, CodeBlock[sp, temp1, 8]
 241     btinz temp1, .copyHeaderLoop
 242
 243     loadi PayloadOffset + ProtoCallFrame::argCountAndCodeOriginValue[protoCallFrame], temp2
 244     subi 1, temp2
 245     loadi ProtoCallFrame::paddedArgCount[protoCallFrame], temp3
 246     subi 1, temp3
 247
 248     bieq temp2, temp3, .copyArgs
 249     move ValueUndefined, temp1
 250 .fillExtraArgsLoop:
 251     subi 1, temp3
 252     storeq temp1, ThisArgumentOffset + 8[sp, temp3, 8]
 253     bineq temp2, temp3, .fillExtraArgsLoop
 254
 255 .copyArgs:
 256     loadp ProtoCallFrame::args[protoCallFrame], temp1
 257
 258 .copyArgsLoop:
 259     btiz temp2, .copyArgsDone
 260     subi 1, temp2
 261     loadq [temp1, temp2, 8], temp3
 262     storeq temp3, ThisArgumentOffset + 8[sp, temp2, 8]
 263     jmp .copyArgsLoop
 264
 265 .copyArgsDone:
 266     if ARM64
 267         move sp, temp2
 268         storep temp2, VM::topCallFrame[vm]
 269     else
 270         storep sp, VM::topCallFrame[vm]
 271     end
 272
 273     move 0xffff000000000000, csr1
 274     addp 2, csr1, csr2
 275
 276     checkStackPointerAlignment(temp3, 0xbad0dc02)
 277
 278     makeCall(entry, temp1)
 279
 280     checkStackPointerAlignment(temp3, 0xbad0dc03)
 281
 282     bpeq CodeBlock[cfr], 1, .calleeFramePopped
 283     loadp CallerFrame[cfr], cfr
 284
 285 .calleeFramePopped:
 286     loadp Callee[cfr], temp2 # VM
 287     loadp ScopeChain[cfr], temp3 # previous topCallFrame
 288     storep temp3, VM::topCallFrame[temp2]
 289
 290     checkStackPointerAlignment(temp3, 0xbad0dc04)
 291
 292     if X86_64 or X86_64_WIN
 293         pop t5
 294     end
 295     callToJavaScriptEpilogue()
 296
 297     ret
 298 end
 299
 300
 301 macro makeJavaScriptCall(entry, temp)
 302     addp 16, sp
 303     if C_LOOP
 304         cloopCallJSFunction entry
 305     else
 306         call entry
 307     end
 308     subp 16, sp
 309 end
 310
 311
 312 macro makeHostFunctionCall(entry, temp)
 313     move entry, temp
 314     if X86_64
 315         move sp, t4
 316     elsif X86_64_WIN
 317         move sp, t2
 318     elsif ARM64 or C_LOOP
 319         move sp, a0
 320     end
 321     if C_LOOP
 322         storep cfr, [sp]
 323         storep lr, 8[sp]
 324         cloopCallNative temp
 325     elsif X86_64_WIN
 326         # For a host function call, JIT relies on that the CallerFrame (frame pointer) is put on the stack,
 327         # On Win64 we need to manually copy the frame pointer to the stack, since MSVC may not maintain a frame pointer on 64-bit.
 328         # See http://msdn.microsoft.com/en-us/library/9z1stfyw.aspx where it's stated that rbp MAY be used as a frame pointer.
 329         storep cfr, [sp]
 330
 331         # We need to allocate 32 bytes on the stack for the shadow space.
 332         subp 32, sp
 333         call temp
 334         addp 32, sp
 335     else
 336         addp 16, sp
 337         call temp
 338         subp 16, sp
 339     end
 340 end
 341
 342
 343 _handleUncaughtException:
 344     loadp ScopeChain[cfr], t3
 345     andp MarkedBlockMask, t3
 346     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
 347     loadp VM::callFrameForThrow[t3], cfr
 348
 349     # So far, we've unwound the stack to the frame just below the sentinel frame, except
 350     # in the case of stack overflow in the first function called from callToJavaScript.
 351     # Check if we need to pop to the sentinel frame and do the necessary clean up for
 352     # returning to the caller C frame.
 353     bpeq CodeBlock[cfr], 1, .handleUncaughtExceptionAlreadyIsSentinel
 354     loadp CallerFrame[cfr], cfr
 355 .handleUncaughtExceptionAlreadyIsSentinel:
 356
 357     loadp Callee[cfr], t3 # VM
 358     loadp ScopeChain[cfr], t5 # previous topCallFrame
 359     storep t5, VM::topCallFrame[t3]
 360
 361     callToJavaScriptEpilogue()
 362     ret
 363
 364
 365 macro prepareStateForCCall()
 366     leap [PB, PC, 8], PC
 367     move PB, t3
 368 end
 369
 370 macro restoreStateAfterCCall()
 371     move t0, PC
 372     move t3, PB
 373     subp PB, PC
 374     rshiftp 3, PC
 375 end
 376
 377 macro callSlowPath(slowPath)
 378     prepareStateForCCall()
 379     cCall2(slowPath, cfr, PC)
 380     restoreStateAfterCCall()
 381 end
 382
 383 macro traceOperand(fromWhere, operand)
 384     prepareStateForCCall()
 385     cCall4(_llint_trace_operand, cfr, PC, fromWhere, operand)
 386     restoreStateAfterCCall()
 387 end
 388
 389 macro traceValue(fromWhere, operand)
 390     prepareStateForCCall()
 391     cCall4(_llint_trace_value, cfr, PC, fromWhere, operand)
 392     restoreStateAfterCCall()
 393 end
 394
 395 # Call a slow path for call call opcodes.
 396 macro callCallSlowPath(slowPath, action)
 397     storei PC, ArgumentCount + TagOffset[cfr]
 398     prepareStateForCCall()
 399     cCall2(slowPath, cfr, PC)
 400     action(t0)
 401 end
 402
 403 macro callWatchdogTimerHandler(throwHandler)
 404     storei PC, ArgumentCount + TagOffset[cfr]
 405     prepareStateForCCall()
 406     cCall2(_llint_slow_path_handle_watchdog_timer, cfr, PC)
 407     btpnz t0, throwHandler
 408     move t3, PB
 409     loadi ArgumentCount + TagOffset[cfr], PC
 410 end
 411
 412 macro checkSwitchToJITForLoop()
 413     checkSwitchToJIT(
 414         1,
 415         macro()
 416             storei PC, ArgumentCount + TagOffset[cfr]
 417             prepareStateForCCall()
 418             cCall2(_llint_loop_osr, cfr, PC)
 419             btpz t0, .recover
 420             move t1, sp
 421             jmp t0
 422         .recover:
 423             move t3, PB
 424             loadi ArgumentCount + TagOffset[cfr], PC
 425         end)
 426 end
 427
 428 macro loadVariable(operand, value)
 429     loadisFromInstruction(operand, value)
 430     loadq [cfr, value, 8], value
 431 end
 432
 433 # Index and value must be different registers. Index may be clobbered.
 434 macro loadConstantOrVariable(index, value)
 435     bpgteq index, FirstConstantRegisterIndex, .constant
 436     loadq [cfr, index, 8], value
 437     jmp .done
 438 .constant:
 439     loadp CodeBlock[cfr], value
 440     loadp CodeBlock::m_constantRegisters + VectorBufferOffset[value], value
 441     subp FirstConstantRegisterIndex, index
 442     loadq [value, index, 8], value
 443 .done:
 444 end
 445
 446 macro loadConstantOrVariableInt32(index, value, slow)
 447     loadConstantOrVariable(index, value)
 448     bqb value, tagTypeNumber, slow
 449 end
 450
 451 macro loadConstantOrVariableCell(index, value, slow)
 452     loadConstantOrVariable(index, value)
 453     btqnz value, tagMask, slow
 454 end
 455
 456 macro writeBarrierOnOperand(cellOperand)
 457     if GGC
 458         loadisFromInstruction(cellOperand, t1)
 459         loadConstantOrVariableCell(t1, t2, .writeBarrierDone)
 460         checkMarkByte(t2, t1, t3,
 461             macro(gcData)
 462                 btbnz gcData, .writeBarrierDone
 463                 push PB, PC
 464                 cCall2Void(_llint_write_barrier_slow, cfr, t2)
 465                 pop PC, PB
 466             end
 467         )
 468     .writeBarrierDone:
 469     end
 470 end
 471
 472 macro writeBarrierOnOperands(cellOperand, valueOperand)
 473     if GGC
 474         loadisFromInstruction(valueOperand, t1)
 475         loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
 476         btpz t0, .writeBarrierDone
 477
 478         writeBarrierOnOperand(cellOperand)
 479     .writeBarrierDone:
 480     end
 481 end
 482
 483 macro writeBarrierOnGlobalObject(valueOperand)
 484     if GGC
 485         loadisFromInstruction(valueOperand, t1)
 486         loadConstantOrVariableCell(t1, t0, .writeBarrierDone)
 487         btpz t0, .writeBarrierDone
 488
 489         loadp CodeBlock[cfr], t3
 490         loadp CodeBlock::m_globalObject[t3], t3
 491         checkMarkByte(t3, t1, t2,
 492             macro(gcData)
 493                 btbnz gcData, .writeBarrierDone
 494                 push PB, PC
 495                 cCall2Void(_llint_write_barrier_slow, cfr, t3)
 496                 pop PC, PB
 497             end
 498         )
 499     .writeBarrierDone:
 500     end
 501 end
 502
 503 macro valueProfile(value, operand, scratch)
 504     loadpFromInstruction(operand, scratch)
 505     storeq value, ValueProfile::m_buckets[scratch]
 506 end
 507
 508 macro loadStructure(cell, structure)
 509 end
 510
 511 macro loadStructureWithScratch(cell, structure, scratch)
 512     loadp CodeBlock[cfr], scratch
 513     loadp CodeBlock::m_vm[scratch], scratch
 514     loadp VM::heap + Heap::m_structureIDTable + StructureIDTable::m_table[scratch], scratch
 515     loadi JSCell::m_structureID[cell], structure
 516     loadp [scratch, structure, 8], structure
 517 end
 518
 519 macro loadStructureAndClobberFirstArg(cell, structure)
 520     loadi JSCell::m_structureID[cell], structure
 521     loadp CodeBlock[cfr], cell
 522     loadp CodeBlock::m_vm[cell], cell
 523     loadp VM::heap + Heap::m_structureIDTable + StructureIDTable::m_table[cell], cell
 524     loadp [cell, structure, 8], structure
 525 end
 526
 527 macro storeStructureWithTypeInfo(cell, structure, scratch)
 528     loadq Structure::m_blob + StructureIDBlob::u.doubleWord[structure], scratch
 529     storeq scratch, JSCell::m_structureID[cell]
 530 end
 531
 532 # Entrypoints into the interpreter.
 533
 534 # Expects that CodeBlock is in t1, which is what prologue() leaves behind.
 535 macro functionArityCheck(doneLabel, slowPath)
 536     loadi PayloadOffset + ArgumentCount[cfr], t0
 537     biaeq t0, CodeBlock::m_numParameters[t1], doneLabel
 538     prepareStateForCCall()
 539     cCall2(slowPath, cfr, PC)   # This slowPath has the protocol: t0 = 0 => no error, t0 != 0 => error
 540     btiz t0, .noError
 541     move t1, cfr   # t1 contains caller frame
 542     jmp _llint_throw_from_slow_path_trampoline
 543
 544 .noError:
 545     # t1 points to ArityCheckData.
 546     loadp CommonSlowPaths::ArityCheckData::thunkToCall[t1], t2
 547     btpz t2, .proceedInline
 548
 549     loadp CommonSlowPaths::ArityCheckData::returnPC[t1], t7
 550     loadp CommonSlowPaths::ArityCheckData::paddedStackSpace[t1], t0
 551     call t2
 552     if ASSERT_ENABLED
 553         loadp ReturnPC[cfr], t0
 554         loadp [t0], t0
 555     end
 556     jmp .continue
 557
 558 .proceedInline:
 559     loadi CommonSlowPaths::ArityCheckData::paddedStackSpace[t1], t1
 560     btiz t1, .continue
 561
 562     // Move frame up "t1 * 2" slots
 563     lshiftp 1, t1
 564     negq t1
 565     move cfr, t3
 566     loadi PayloadOffset + ArgumentCount[cfr], t2
 567     addi CallFrameHeaderSlots, t2
 568 .copyLoop:
 569     loadq [t3], t0
 570     storeq t0, [t3, t1, 8]
 571     addp 8, t3
 572     bsubinz 1, t2, .copyLoop
 573
 574     // Fill new slots with JSUndefined
 575     move t1, t2
 576     move ValueUndefined, t0
 577 .fillLoop:
 578     storeq t0, [t3, t1, 8]
 579     addp 8, t3
 580     baddinz 1, t2, .fillLoop
 581
 582     lshiftp 3, t1
 583     addp t1, cfr
 584     addp t1, sp
 585
 586 .continue:
 587     # Reload CodeBlock and reset PC, since the slow_path clobbered them.
 588     loadp CodeBlock[cfr], t1
 589     loadp CodeBlock::m_instructions[t1], PB
 590     move 0, PC
 591     jmp doneLabel
 592 end
 593
 594 macro branchIfException(label)
 595     loadp ScopeChain[cfr], t3
 596     andp MarkedBlockMask, t3
 597     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
 598     btqz VM::m_exception[t3], .noException
 599     jmp label
 600 .noException:
 601 end
 602
 603
 604 # Instruction implementations
 605
 606 _llint_op_enter:
 607     traceExecution()
 608     checkStackPointerAlignment(t2, 0xdead00e1)
 609     loadp CodeBlock[cfr], t2                // t2<CodeBlock> = cfr.CodeBlock
 610     loadi CodeBlock::m_numVars[t2], t2      // t2<size_t> = t2<CodeBlock>.m_numVars
 611     btiz t2, .opEnterDone
 612     move ValueUndefined, t0
 613     negi t2
 614     sxi2q t2, t2
 615 .opEnterLoop:
 616     storeq t0, [cfr, t2, 8]
 617     addq 1, t2
 618     btqnz t2, .opEnterLoop
 619 .opEnterDone:
 620     callSlowPath(_slow_path_enter)
 621     dispatch(1)
 622
 623
 624 _llint_op_create_activation:
 625     traceExecution()
 626     loadisFromInstruction(1, t0)
 627     bqneq [cfr, t0, 8], ValueEmpty, .opCreateActivationDone
 628     callSlowPath(_llint_slow_path_create_activation)
 629 .opCreateActivationDone:
 630     dispatch(2)
 631
 632
 633 _llint_op_init_lazy_reg:
 634     traceExecution()
 635     loadisFromInstruction(1, t0)
 636     storeq ValueEmpty, [cfr, t0, 8]
 637     dispatch(2)
 638
 639
 640 _llint_op_create_arguments:
 641     traceExecution()
 642     loadisFromInstruction(1, t0)
 643     bqneq [cfr, t0, 8], ValueEmpty, .opCreateArgumentsDone
 644     callSlowPath(_slow_path_create_arguments)
 645 .opCreateArgumentsDone:
 646     dispatch(2)
 647
 648
 649 _llint_op_create_this:
 650     traceExecution()
 651     loadisFromInstruction(2, t0)
 652     loadp [cfr, t0, 8], t0
 653     loadp JSFunction::m_allocationProfile + ObjectAllocationProfile::m_allocator[t0], t1
 654     loadp JSFunction::m_allocationProfile + ObjectAllocationProfile::m_structure[t0], t2
 655     btpz t1, .opCreateThisSlow
 656     allocateJSObject(t1, t2, t0, t3, .opCreateThisSlow)
 657     loadisFromInstruction(1, t1)
 658     storeq t0, [cfr, t1, 8]
 659     dispatch(4)
 660
 661 .opCreateThisSlow:
 662     callSlowPath(_slow_path_create_this)
 663     dispatch(4)
 664
 665
 666 _llint_op_get_callee:
 667     traceExecution()
 668     loadisFromInstruction(1, t0)
 669     loadp Callee[cfr], t1
 670     loadpFromInstruction(2, t2)
 671     bpneq t1, t2, .opGetCalleeSlow
 672     storep t1, [cfr, t0, 8]
 673     dispatch(3)
 674
 675 .opGetCalleeSlow:
 676     callSlowPath(_slow_path_get_callee)
 677     dispatch(3)
 678
 679 _llint_op_to_this:
 680     traceExecution()
 681     loadisFromInstruction(1, t0)
 682     loadq [cfr, t0, 8], t0
 683     btqnz t0, tagMask, .opToThisSlow
 684     bbneq JSCell::m_type[t0], FinalObjectType, .opToThisSlow
 685     loadStructureWithScratch(t0, t1, t2)
 686     loadpFromInstruction(2, t2)
 687     bpneq t1, t2, .opToThisSlow
 688     dispatch(3)
 689
 690 .opToThisSlow:
 691     callSlowPath(_slow_path_to_this)
 692     dispatch(3)
 693
 694
 695 _llint_op_new_object:
 696     traceExecution()
 697     loadpFromInstruction(3, t0)
 698     loadp ObjectAllocationProfile::m_allocator[t0], t1
 699     loadp ObjectAllocationProfile::m_structure[t0], t2
 700     allocateJSObject(t1, t2, t0, t3, .opNewObjectSlow)
 701     loadisFromInstruction(1, t1)
 702     storeq t0, [cfr, t1, 8]
 703     dispatch(4)
 704
 705 .opNewObjectSlow:
 706     callSlowPath(_llint_slow_path_new_object)
 707     dispatch(4)
 708
 709
 710 _llint_op_mov:
 711     traceExecution()
 712     loadisFromInstruction(2, t1)
 713     loadisFromInstruction(1, t0)
 714     loadConstantOrVariable(t1, t2)
 715     storeq t2, [cfr, t0, 8]
 716     dispatch(3)
 717
 718
 719 macro notifyWrite(set, value, scratch, slow)
 720     loadb VariableWatchpointSet::m_state[set], scratch
 721     bieq scratch, IsInvalidated, .done
 722     bqneq value, VariableWatchpointSet::m_inferredValue[set], slow
 723 .done:
 724 end
 725
 726 _llint_op_captured_mov:
 727     traceExecution()
 728     loadisFromInstruction(2, t1)
 729     loadConstantOrVariable(t1, t2)
 730     loadpFromInstruction(3, t0)
 731     btpz t0, .opCapturedMovReady
 732     notifyWrite(t0, t2, t1, .opCapturedMovSlow)
 733 .opCapturedMovReady:
 734     loadisFromInstruction(1, t0)
 735     storeq t2, [cfr, t0, 8]
 736     dispatch(4)
 737
 738 .opCapturedMovSlow:
 739     callSlowPath(_slow_path_captured_mov)
 740     dispatch(4)
 741
 742
 743 _llint_op_not:
 744     traceExecution()
 745     loadisFromInstruction(2, t0)
 746     loadisFromInstruction(1, t1)
 747     loadConstantOrVariable(t0, t2)
 748     xorq ValueFalse, t2
 749     btqnz t2, ~1, .opNotSlow
 750     xorq ValueTrue, t2
 751     storeq t2, [cfr, t1, 8]
 752     dispatch(3)
 753
 754 .opNotSlow:
 755     callSlowPath(_slow_path_not)
 756     dispatch(3)
 757
 758
 759 macro equalityComparison(integerComparison, slowPath)
 760     traceExecution()
 761     loadisFromInstruction(3, t0)
 762     loadisFromInstruction(2, t2)
 763     loadisFromInstruction(1, t3)
 764     loadConstantOrVariableInt32(t0, t1, .slow)
 765     loadConstantOrVariableInt32(t2, t0, .slow)
 766     integerComparison(t0, t1, t0)
 767     orq ValueFalse, t0
 768     storeq t0, [cfr, t3, 8]
 769     dispatch(4)
 770
 771 .slow:
 772     callSlowPath(slowPath)
 773     dispatch(4)
 774 end
 775
 776 _llint_op_eq:
 777     equalityComparison(
 778         macro (left, right, result) cieq left, right, result end,
 779         _slow_path_eq)
 780
 781
 782 _llint_op_neq:
 783     equalityComparison(
 784         macro (left, right, result) cineq left, right, result end,
 785         _slow_path_neq)
 786
 787
 788 macro equalNullComparison()
 789     loadisFromInstruction(2, t0)
 790     loadq [cfr, t0, 8], t0
 791     btqnz t0, tagMask, .immediate
 792     btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined
 793     move 0, t0
 794     jmp .done
 795 .masqueradesAsUndefined:
 796     loadStructureWithScratch(t0, t2, t1)
 797     loadp CodeBlock[cfr], t0
 798     loadp CodeBlock::m_globalObject[t0], t0
 799     cpeq Structure::m_globalObject[t2], t0, t0
 800     jmp .done
 801 .immediate:
 802     andq ~TagBitUndefined, t0
 803     cqeq t0, ValueNull, t0
 804 .done:
 805 end
 806
 807 _llint_op_eq_null:
 808     traceExecution()
 809     equalNullComparison()
 810     loadisFromInstruction(1, t1)
 811     orq ValueFalse, t0
 812     storeq t0, [cfr, t1, 8]
 813     dispatch(3)
 814
 815
 816 _llint_op_neq_null:
 817     traceExecution()
 818     equalNullComparison()
 819     loadisFromInstruction(1, t1)
 820     xorq ValueTrue, t0
 821     storeq t0, [cfr, t1, 8]
 822     dispatch(3)
 823
 824
 825 macro strictEq(equalityOperation, slowPath)
 826     traceExecution()
 827     loadisFromInstruction(3, t0)
 828     loadisFromInstruction(2, t2)
 829     loadConstantOrVariable(t0, t1)
 830     loadConstantOrVariable(t2, t0)
 831     move t0, t2
 832     orq t1, t2
 833     btqz t2, tagMask, .slow
 834     bqaeq t0, tagTypeNumber, .leftOK
 835     btqnz t0, tagTypeNumber, .slow
 836 .leftOK:
 837     bqaeq t1, tagTypeNumber, .rightOK
 838     btqnz t1, tagTypeNumber, .slow
 839 .rightOK:
 840     equalityOperation(t0, t1, t0)
 841     loadisFromInstruction(1, t1)
 842     orq ValueFalse, t0
 843     storeq t0, [cfr, t1, 8]
 844     dispatch(4)
 845
 846 .slow:
 847     callSlowPath(slowPath)
 848     dispatch(4)
 849 end
 850
 851 _llint_op_stricteq:
 852     strictEq(
 853         macro (left, right, result) cqeq left, right, result end,
 854         _slow_path_stricteq)
 855
 856
 857 _llint_op_nstricteq:
 858     strictEq(
 859         macro (left, right, result) cqneq left, right, result end,
 860         _slow_path_nstricteq)
 861
 862
 863 macro preOp(arithmeticOperation, slowPath)
 864     traceExecution()
 865     loadisFromInstruction(1, t0)
 866     loadq [cfr, t0, 8], t1
 867     bqb t1, tagTypeNumber, .slow
 868     arithmeticOperation(t1, .slow)
 869     orq tagTypeNumber, t1
 870     storeq t1, [cfr, t0, 8]
 871     dispatch(2)
 872
 873 .slow:
 874     callSlowPath(slowPath)
 875     dispatch(2)
 876 end
 877
 878 _llint_op_inc:
 879     preOp(
 880         macro (value, slow) baddio 1, value, slow end,
 881         _slow_path_inc)
 882
 883
 884 _llint_op_dec:
 885     preOp(
 886         macro (value, slow) bsubio 1, value, slow end,
 887         _slow_path_dec)
 888
 889
 890 _llint_op_to_number:
 891     traceExecution()
 892     loadisFromInstruction(2, t0)
 893     loadisFromInstruction(1, t1)
 894     loadConstantOrVariable(t0, t2)
 895     bqaeq t2, tagTypeNumber, .opToNumberIsImmediate
 896     btqz t2, tagTypeNumber, .opToNumberSlow
 897 .opToNumberIsImmediate:
 898     storeq t2, [cfr, t1, 8]
 899     dispatch(3)
 900
 901 .opToNumberSlow:
 902     callSlowPath(_slow_path_to_number)
 903     dispatch(3)
 904
 905
 906 _llint_op_negate:
 907     traceExecution()
 908     loadisFromInstruction(2, t0)
 909     loadisFromInstruction(1, t1)
 910     loadConstantOrVariable(t0, t2)
 911     bqb t2, tagTypeNumber, .opNegateNotInt
 912     btiz t2, 0x7fffffff, .opNegateSlow
 913     negi t2
 914     orq tagTypeNumber, t2
 915     storeq t2, [cfr, t1, 8]
 916     dispatch(3)
 917 .opNegateNotInt:
 918     btqz t2, tagTypeNumber, .opNegateSlow
 919     xorq 0x8000000000000000, t2
 920     storeq t2, [cfr, t1, 8]
 921     dispatch(3)
 922
 923 .opNegateSlow:
 924     callSlowPath(_slow_path_negate)
 925     dispatch(3)
 926
 927
 928 macro binaryOpCustomStore(integerOperationAndStore, doubleOperation, slowPath)
 929     loadisFromInstruction(3, t0)
 930     loadisFromInstruction(2, t2)
 931     loadConstantOrVariable(t0, t1)
 932     loadConstantOrVariable(t2, t0)
 933     bqb t0, tagTypeNumber, .op1NotInt
 934     bqb t1, tagTypeNumber, .op2NotInt
 935     loadisFromInstruction(1, t2)
 936     integerOperationAndStore(t1, t0, .slow, t2)
 937     dispatch(5)
 938
 939 .op1NotInt:
 940     # First operand is definitely not an int, the second operand could be anything.
 941     btqz t0, tagTypeNumber, .slow
 942     bqaeq t1, tagTypeNumber, .op1NotIntOp2Int
 943     btqz t1, tagTypeNumber, .slow
 944     addq tagTypeNumber, t1
 945     fq2d t1, ft1
 946     jmp .op1NotIntReady
 947 .op1NotIntOp2Int:
 948     ci2d t1, ft1
 949 .op1NotIntReady:
 950     loadisFromInstruction(1, t2)
 951     addq tagTypeNumber, t0
 952     fq2d t0, ft0
 953     doubleOperation(ft1, ft0)
 954     fd2q ft0, t0
 955     subq tagTypeNumber, t0
 956     storeq t0, [cfr, t2, 8]
 957     dispatch(5)
 958
 959 .op2NotInt:
 960     # First operand is definitely an int, the second is definitely not.
 961     loadisFromInstruction(1, t2)
 962     btqz t1, tagTypeNumber, .slow
 963     ci2d t0, ft0
 964     addq tagTypeNumber, t1
 965     fq2d t1, ft1
 966     doubleOperation(ft1, ft0)
 967     fd2q ft0, t0
 968     subq tagTypeNumber, t0
 969     storeq t0, [cfr, t2, 8]
 970     dispatch(5)
 971
 972 .slow:
 973     callSlowPath(slowPath)
 974     dispatch(5)
 975 end
 976
 977 macro binaryOp(integerOperation, doubleOperation, slowPath)
 978     binaryOpCustomStore(
 979         macro (left, right, slow, index)
 980             integerOperation(left, right, slow)
 981             orq tagTypeNumber, right
 982             storeq right, [cfr, index, 8]
 983         end,
 984         doubleOperation, slowPath)
 985 end
 986
 987 _llint_op_add:
 988     traceExecution()
 989     binaryOp(
 990         macro (left, right, slow) baddio left, right, slow end,
 991         macro (left, right) addd left, right end,
 992         _slow_path_add)
 993
 994
 995 _llint_op_mul:
 996     traceExecution()
 997     binaryOpCustomStore(
 998         macro (left, right, slow, index)
 999             # Assume t3 is scratchable.
1000             move right, t3
1001             bmulio left, t3, slow
1002             btinz t3, .done
1003             bilt left, 0, slow
1004             bilt right, 0, slow
1005         .done:
1006             orq tagTypeNumber, t3
1007             storeq t3, [cfr, index, 8]
1008         end,
1009         macro (left, right) muld left, right end,
1010         _slow_path_mul)
1011
1012
1013 _llint_op_sub:
1014     traceExecution()
1015     binaryOp(
1016         macro (left, right, slow) bsubio left, right, slow end,
1017         macro (left, right) subd left, right end,
1018         _slow_path_sub)
1019
1020
1021 _llint_op_div:
1022     traceExecution()
1023     if X86_64 or X86_64_WIN
1024         binaryOpCustomStore(
1025             macro (left, right, slow, index)
1026                 # Assume t3 is scratchable.
1027                 btiz left, slow
1028                 bineq left, -1, .notNeg2TwoThe31DivByNeg1
1029                 bieq right, -2147483648, .slow
1030             .notNeg2TwoThe31DivByNeg1:
1031                 btinz right, .intOK
1032                 bilt left, 0, slow
1033             .intOK:
1034                 move left, t3
1035                 move right, t0
1036                 cdqi
1037                 idivi t3
1038                 btinz t1, slow
1039                 orq tagTypeNumber, t0
1040                 storeq t0, [cfr, index, 8]
1041             end,
1042             macro (left, right) divd left, right end,
1043             _slow_path_div)
1044     else
1045         callSlowPath(_slow_path_div)
1046         dispatch(5)
1047     end
1048
1049
1050 macro bitOp(operation, slowPath, advance)
1051     loadisFromInstruction(3, t0)
1052     loadisFromInstruction(2, t2)
1053     loadisFromInstruction(1, t3)
1054     loadConstantOrVariable(t0, t1)
1055     loadConstantOrVariable(t2, t0)
1056     bqb t0, tagTypeNumber, .slow
1057     bqb t1, tagTypeNumber, .slow
1058     operation(t1, t0)
1059     orq tagTypeNumber, t0
1060     storeq t0, [cfr, t3, 8]
1061     dispatch(advance)
1062
1063 .slow:
1064     callSlowPath(slowPath)
1065     dispatch(advance)
1066 end
1067
1068 _llint_op_lshift:
1069     traceExecution()
1070     bitOp(
1071         macro (left, right) lshifti left, right end,
1072         _slow_path_lshift,
1073         4)
1074
1075
1076 _llint_op_rshift:
1077     traceExecution()
1078     bitOp(
1079         macro (left, right) rshifti left, right end,
1080         _slow_path_rshift,
1081         4)
1082
1083
1084 _llint_op_urshift:
1085     traceExecution()
1086     bitOp(
1087         macro (left, right) urshifti left, right end,
1088         _slow_path_urshift,
1089         4)
1090
1091
1092 _llint_op_unsigned:
1093     traceExecution()
1094     loadisFromInstruction(1, t0)
1095     loadisFromInstruction(2, t1)
1096     loadConstantOrVariable(t1, t2)
1097     bilt t2, 0, .opUnsignedSlow
1098     storeq t2, [cfr, t0, 8]
1099     dispatch(3)
1100 .opUnsignedSlow:
1101     callSlowPath(_slow_path_unsigned)
1102     dispatch(3)
1103
1104
1105 _llint_op_bitand:
1106     traceExecution()
1107     bitOp(
1108         macro (left, right) andi left, right end,
1109         _slow_path_bitand,
1110         5)
1111
1112
1113 _llint_op_bitxor:
1114     traceExecution()
1115     bitOp(
1116         macro (left, right) xori left, right end,
1117         _slow_path_bitxor,
1118         5)
1119
1120
1121 _llint_op_bitor:
1122     traceExecution()
1123     bitOp(
1124         macro (left, right) ori left, right end,
1125         _slow_path_bitor,
1126         5)
1127
1128
1129 _llint_op_check_has_instance:
1130     traceExecution()
1131     loadisFromInstruction(3, t1)
1132     loadConstantOrVariableCell(t1, t0, .opCheckHasInstanceSlow)
1133     btbz JSCell::m_flags[t0], ImplementsDefaultHasInstance, .opCheckHasInstanceSlow
1134     dispatch(5)
1135
1136 .opCheckHasInstanceSlow:
1137     callSlowPath(_llint_slow_path_check_has_instance)
1138     dispatch(0)
1139
1140
1141 _llint_op_instanceof:
1142     traceExecution()
1143     # Actually do the work.
1144     loadisFromInstruction(3, t0)
1145     loadConstantOrVariableCell(t0, t1, .opInstanceofSlow)
1146     bbb JSCell::m_type[t1], ObjectType, .opInstanceofSlow
1147     loadisFromInstruction(2, t0)
1148     loadConstantOrVariableCell(t0, t2, .opInstanceofSlow)
1149
1150     # Register state: t1 = prototype, t2 = value
1151     move 1, t0
1152 .opInstanceofLoop:
1153     loadStructureAndClobberFirstArg(t2, t3)
1154     loadq Structure::m_prototype[t3], t2
1155     bqeq t2, t1, .opInstanceofDone
1156     btqz t2, tagMask, .opInstanceofLoop
1157
1158     move 0, t0
1159 .opInstanceofDone:
1160     orq ValueFalse, t0
1161     loadisFromInstruction(1, t3)
1162     storeq t0, [cfr, t3, 8]
1163     dispatch(4)
1164
1165 .opInstanceofSlow:
1166     callSlowPath(_llint_slow_path_instanceof)
1167     dispatch(4)
1168
1169
1170 _llint_op_is_undefined:
1171     traceExecution()
1172     loadisFromInstruction(2, t1)
1173     loadisFromInstruction(1, t2)
1174     loadConstantOrVariable(t1, t0)
1175     btqz t0, tagMask, .opIsUndefinedCell
1176     cqeq t0, ValueUndefined, t3
1177     orq ValueFalse, t3
1178     storeq t3, [cfr, t2, 8]
1179     dispatch(3)
1180 .opIsUndefinedCell:
1181     btbnz JSCell::m_flags[t0], MasqueradesAsUndefined, .masqueradesAsUndefined
1182     move ValueFalse, t1
1183     storeq t1, [cfr, t2, 8]
1184     dispatch(3)
1185 .masqueradesAsUndefined:
1186     loadStructureWithScratch(t0, t3, t1)
1187     loadp CodeBlock[cfr], t1
1188     loadp CodeBlock::m_globalObject[t1], t1
1189     cpeq Structure::m_globalObject[t3], t1, t0
1190     orq ValueFalse, t0
1191     storeq t0, [cfr, t2, 8]
1192     dispatch(3)
1193
1194
1195 _llint_op_is_boolean:
1196     traceExecution()
1197     loadisFromInstruction(2, t1)
1198     loadisFromInstruction(1, t2)
1199     loadConstantOrVariable(t1, t0)
1200     xorq ValueFalse, t0
1201     tqz t0, ~1, t0
1202     orq ValueFalse, t0
1203     storeq t0, [cfr, t2, 8]
1204     dispatch(3)
1205
1206
1207 _llint_op_is_number:
1208     traceExecution()
1209     loadisFromInstruction(2, t1)
1210     loadisFromInstruction(1, t2)
1211     loadConstantOrVariable(t1, t0)
1212     tqnz t0, tagTypeNumber, t1
1213     orq ValueFalse, t1
1214     storeq t1, [cfr, t2, 8]
1215     dispatch(3)
1216
1217
1218 _llint_op_is_string:
1219     traceExecution()
1220     loadisFromInstruction(2, t1)
1221     loadisFromInstruction(1, t2)
1222     loadConstantOrVariable(t1, t0)
1223     btqnz t0, tagMask, .opIsStringNotCell
1224     cbeq JSCell::m_type[t0], StringType, t1
1225     orq ValueFalse, t1
1226     storeq t1, [cfr, t2, 8]
1227     dispatch(3)
1228 .opIsStringNotCell:
1229     storeq ValueFalse, [cfr, t2, 8]
1230     dispatch(3)
1231
1232
1233 macro loadPropertyAtVariableOffset(propertyOffsetAsInt, objectAndStorage, value)
1234     bilt propertyOffsetAsInt, firstOutOfLineOffset, .isInline
1235     loadp JSObject::m_butterfly[objectAndStorage], objectAndStorage
1236     negi propertyOffsetAsInt
1237     sxi2q propertyOffsetAsInt, propertyOffsetAsInt
1238     jmp .ready
1239 .isInline:
1240     addp sizeof JSObject - (firstOutOfLineOffset - 2) * 8, objectAndStorage
1241 .ready:
1242     loadq (firstOutOfLineOffset - 2) * 8[objectAndStorage, propertyOffsetAsInt, 8], value
1243 end
1244
1245
1246 macro storePropertyAtVariableOffset(propertyOffsetAsInt, objectAndStorage, value)
1247     bilt propertyOffsetAsInt, firstOutOfLineOffset, .isInline
1248     loadp JSObject::m_butterfly[objectAndStorage], objectAndStorage
1249     negi propertyOffsetAsInt
1250     sxi2q propertyOffsetAsInt, propertyOffsetAsInt
1251     jmp .ready
1252 .isInline:
1253     addp sizeof JSObject - (firstOutOfLineOffset - 2) * 8, objectAndStorage
1254 .ready:
1255     storeq value, (firstOutOfLineOffset - 2) * 8[objectAndStorage, propertyOffsetAsInt, 8]
1256 end
1257
1258 _llint_op_init_global_const:
1259     traceExecution()
1260     writeBarrierOnGlobalObject(2)
1261     loadisFromInstruction(2, t1)
1262     loadpFromInstruction(1, t0)
1263     loadConstantOrVariable(t1, t2)
1264     storeq t2, [t0]
1265     dispatch(5)
1266
1267
1268 macro getById(getPropertyStorage)
1269     traceExecution()
1270     # We only do monomorphic get_by_id caching for now, and we do not modify the
1271     # opcode. We do, however, allow for the cache to change anytime if fails, since
1272     # ping-ponging is free. At best we get lucky and the get_by_id will continue
1273     # to take fast path on the new cache. At worst we take slow path, which is what
1274     # we would have been doing anyway.
1275     loadisFromInstruction(2, t0)
1276     loadConstantOrVariableCell(t0, t3, .opGetByIdSlow)
1277     loadStructureWithScratch(t3, t2, t1)
1278     loadpFromInstruction(4, t1)
1279     bpneq t2, t1, .opGetByIdSlow
1280     getPropertyStorage(
1281         t3,
1282         t0,
1283         macro (propertyStorage, scratch)
1284             loadisFromInstruction(5, t2)
1285             loadisFromInstruction(1, t1)
1286             loadq [propertyStorage, t2], scratch
1287             storeq scratch, [cfr, t1, 8]
1288             valueProfile(scratch, 8, t1)
1289             dispatch(9)
1290         end)
1291
1292     .opGetByIdSlow:
1293         callSlowPath(_llint_slow_path_get_by_id)
1294         dispatch(9)
1295 end
1296
1297 _llint_op_get_by_id:
1298     getById(withInlineStorage)
1299
1300
1301 _llint_op_get_by_id_out_of_line:
1302     getById(withOutOfLineStorage)
1303
1304
1305 _llint_op_get_array_length:
1306     traceExecution()
1307     loadisFromInstruction(2, t0)
1308     loadpFromInstruction(4, t1)
1309     loadConstantOrVariableCell(t0, t3, .opGetArrayLengthSlow)
1310     move t3, t2
1311     arrayProfile(t2, t1, t0)
1312     btiz t2, IsArray, .opGetArrayLengthSlow
1313     btiz t2, IndexingShapeMask, .opGetArrayLengthSlow
1314     loadisFromInstruction(1, t1)
1315     loadp JSObject::m_butterfly[t3], t0
1316     loadi -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], t0
1317     bilt t0, 0, .opGetArrayLengthSlow
1318     orq tagTypeNumber, t0
1319     valueProfile(t0, 8, t2)
1320     storeq t0, [cfr, t1, 8]
1321     dispatch(9)
1322
1323 .opGetArrayLengthSlow:
1324     callSlowPath(_llint_slow_path_get_by_id)
1325     dispatch(9)
1326
1327
1328 _llint_op_get_arguments_length:
1329     traceExecution()
1330     loadisFromInstruction(2, t0)
1331     loadisFromInstruction(1, t1)
1332     btqnz [cfr, t0, 8], .opGetArgumentsLengthSlow
1333     loadi ArgumentCount + PayloadOffset[cfr], t2
1334     subi 1, t2
1335     orq tagTypeNumber, t2
1336     storeq t2, [cfr, t1, 8]
1337     dispatch(4)
1338
1339 .opGetArgumentsLengthSlow:
1340     callSlowPath(_llint_slow_path_get_arguments_length)
1341     dispatch(4)
1342
1343
1344 macro putById(getPropertyStorage)
1345     traceExecution()
1346     writeBarrierOnOperands(1, 3)
1347     loadisFromInstruction(1, t3)
1348     loadConstantOrVariableCell(t3, t0, .opPutByIdSlow)
1349     loadStructureWithScratch(t0, t2, t1)
1350     loadpFromInstruction(4, t1)
1351     bpneq t2, t1, .opPutByIdSlow
1352     getPropertyStorage(
1353         t0,
1354         t3,
1355         macro (propertyStorage, scratch)
1356             loadisFromInstruction(5, t1)
1357             loadisFromInstruction(3, t2)
1358             loadConstantOrVariable(t2, scratch)
1359             storeq scratch, [propertyStorage, t1]
1360             dispatch(9)
1361         end)
1362 end
1363
1364 _llint_op_put_by_id:
1365     putById(withInlineStorage)
1366
1367 .opPutByIdSlow:
1368     callSlowPath(_llint_slow_path_put_by_id)
1369     dispatch(9)
1370
1371
1372 _llint_op_put_by_id_out_of_line:
1373     putById(withOutOfLineStorage)
1374
1375
1376 macro putByIdTransition(additionalChecks, getPropertyStorage)
1377     traceExecution()
1378     writeBarrierOnOperand(1)
1379     loadisFromInstruction(1, t3)
1380     loadpFromInstruction(4, t1)
1381     loadConstantOrVariableCell(t3, t0, .opPutByIdSlow)
1382     loadStructureWithScratch(t0, t2, t3)
1383     bpneq t2, t1, .opPutByIdSlow
1384     additionalChecks(t1, t3, t2)
1385     loadisFromInstruction(3, t2)
1386     loadisFromInstruction(5, t1)
1387     getPropertyStorage(
1388         t0,
1389         t3,
1390         macro (propertyStorage, scratch)
1391             addp t1, propertyStorage, t3
1392             loadConstantOrVariable(t2, t1)
1393             storeq t1, [t3]
1394             loadpFromInstruction(6, t1)
1395             loadi Structure::m_blob + StructureIDBlob::u.words.word1[t1], t1
1396             storei t1, JSCell::m_structureID[t0]
1397             dispatch(9)
1398         end)
1399 end
1400
1401 macro noAdditionalChecks(oldStructure, scratch, scratch2)
1402 end
1403
1404 macro structureChainChecks(oldStructure, scratch, scratch2)
1405     const protoCell = oldStructure    # Reusing the oldStructure register for the proto
1406     loadpFromInstruction(7, scratch)
1407     assert(macro (ok) btpnz scratch, ok end)
1408     loadp StructureChain::m_vector[scratch], scratch
1409     assert(macro (ok) btpnz scratch, ok end)
1410     bqeq Structure::m_prototype[oldStructure], ValueNull, .done
1411 .loop:
1412     loadq Structure::m_prototype[oldStructure], protoCell
1413     loadStructureAndClobberFirstArg(protoCell, scratch2)
1414     move scratch2, oldStructure
1415     bpneq oldStructure, [scratch], .opPutByIdSlow
1416     addp 8, scratch
1417     bqneq Structure::m_prototype[oldStructure], ValueNull, .loop
1418 .done:
1419 end
1420
1421 _llint_op_put_by_id_transition_direct:
1422     putByIdTransition(noAdditionalChecks, withInlineStorage)
1423
1424
1425 _llint_op_put_by_id_transition_direct_out_of_line:
1426     putByIdTransition(noAdditionalChecks, withOutOfLineStorage)
1427
1428
1429 _llint_op_put_by_id_transition_normal:
1430     putByIdTransition(structureChainChecks, withInlineStorage)
1431
1432
1433 _llint_op_put_by_id_transition_normal_out_of_line:
1434     putByIdTransition(structureChainChecks, withOutOfLineStorage)
1435
1436
1437 _llint_op_get_by_val:
1438     traceExecution()
1439     loadisFromInstruction(2, t2)
1440     loadConstantOrVariableCell(t2, t0, .opGetByValSlow)
1441     loadpFromInstruction(4, t3)
1442     move t0, t2
1443     arrayProfile(t2, t3, t1)
1444     loadisFromInstruction(3, t3)
1445     loadConstantOrVariableInt32(t3, t1, .opGetByValSlow)
1446     sxi2q t1, t1
1447     loadp JSObject::m_butterfly[t0], t3
1448     andi IndexingShapeMask, t2
1449     bieq t2, Int32Shape, .opGetByValIsContiguous
1450     bineq t2, ContiguousShape, .opGetByValNotContiguous
1451 .opGetByValIsContiguous:
1452
1453     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t3], .opGetByValOutOfBounds
1454     loadisFromInstruction(1, t0)
1455     loadq [t3, t1, 8], t2
1456     btqz t2, .opGetByValOutOfBounds
1457     jmp .opGetByValDone
1458
1459 .opGetByValNotContiguous:
1460     bineq t2, DoubleShape, .opGetByValNotDouble
1461     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t3], .opGetByValOutOfBounds
1462     loadis 8[PB, PC, 8], t0
1463     loadd [t3, t1, 8], ft0
1464     bdnequn ft0, ft0, .opGetByValOutOfBounds
1465     fd2q ft0, t2
1466     subq tagTypeNumber, t2
1467     jmp .opGetByValDone
1468
1469 .opGetByValNotDouble:
1470     subi ArrayStorageShape, t2
1471     bia t2, SlowPutArrayStorageShape - ArrayStorageShape, .opGetByValSlow
1472     biaeq t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t3], .opGetByValOutOfBounds
1473     loadisFromInstruction(1, t0)
1474     loadq ArrayStorage::m_vector[t3, t1, 8], t2
1475     btqz t2, .opGetByValOutOfBounds
1476
1477 .opGetByValDone:
1478     storeq t2, [cfr, t0, 8]
1479     valueProfile(t2, 5, t0)
1480     dispatch(6)
1481
1482 .opGetByValOutOfBounds:
1483     loadpFromInstruction(4, t0)
1484     storeb 1, ArrayProfile::m_outOfBounds[t0]
1485 .opGetByValSlow:
1486     callSlowPath(_llint_slow_path_get_by_val)
1487     dispatch(6)
1488
1489
1490 _llint_op_get_argument_by_val:
1491     # FIXME: At some point we should array profile this. Right now it isn't necessary
1492     # since the DFG will never turn a get_argument_by_val into a GetByVal.
1493     traceExecution()
1494     loadisFromInstruction(2, t0)
1495     loadisFromInstruction(3, t1)
1496     btqnz [cfr, t0, 8], .opGetArgumentByValSlow
1497     loadConstantOrVariableInt32(t1, t2, .opGetArgumentByValSlow)
1498     addi 1, t2
1499     loadi ArgumentCount + PayloadOffset[cfr], t1
1500     biaeq t2, t1, .opGetArgumentByValSlow
1501     loadisFromInstruction(1, t3)
1502     loadpFromInstruction(5, t1)
1503     loadq ThisArgumentOffset[cfr, t2, 8], t0
1504     storeq t0, [cfr, t3, 8]
1505     valueProfile(t0, 5, t1)
1506     dispatch(6)
1507
1508 .opGetArgumentByValSlow:
1509     callSlowPath(_llint_slow_path_get_argument_by_val)
1510     dispatch(6)
1511
1512
1513 _llint_op_get_by_pname:
1514     traceExecution()
1515     loadisFromInstruction(3, t1)
1516     loadConstantOrVariable(t1, t0)
1517     loadisFromInstruction(4, t1)
1518     assertNotConstant(t1)
1519     bqneq t0, [cfr, t1, 8], .opGetByPnameSlow
1520     loadisFromInstruction(2, t2)
1521     loadisFromInstruction(5, t3)
1522     loadConstantOrVariableCell(t2, t0, .opGetByPnameSlow)
1523     assertNotConstant(t3)
1524     loadq [cfr, t3, 8], t1
1525     loadStructureWithScratch(t0, t2, t3)
1526     bpneq t2, JSPropertyNameIterator::m_cachedStructure[t1], .opGetByPnameSlow
1527     loadisFromInstruction(6, t3)
1528     loadi PayloadOffset[cfr, t3, 8], t3
1529     subi 1, t3
1530     biaeq t3, JSPropertyNameIterator::m_numCacheableSlots[t1], .opGetByPnameSlow
1531     bilt t3, JSPropertyNameIterator::m_cachedStructureInlineCapacity[t1], .opGetByPnameInlineProperty
1532     addi firstOutOfLineOffset, t3
1533     subi JSPropertyNameIterator::m_cachedStructureInlineCapacity[t1], t3
1534 .opGetByPnameInlineProperty:
1535     loadPropertyAtVariableOffset(t3, t0, t0)
1536     loadisFromInstruction(1, t1)
1537     storeq t0, [cfr, t1, 8]
1538     dispatch(7)
1539
1540 .opGetByPnameSlow:
1541     callSlowPath(_llint_slow_path_get_by_pname)
1542     dispatch(7)
1543
1544
1545 macro contiguousPutByVal(storeCallback)
1546     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], .outOfBounds
1547 .storeResult:
1548     loadisFromInstruction(3, t2)
1549     storeCallback(t2, t1, [t0, t3, 8])
1550     dispatch(5)
1551
1552 .outOfBounds:
1553     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t0], .opPutByValOutOfBounds
1554     loadp 32[PB, PC, 8], t2
1555     storeb 1, ArrayProfile::m_mayStoreToHole[t2]
1556     addi 1, t3, t2
1557     storei t2, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0]
1558     jmp .storeResult
1559 end
1560
1561 macro putByVal(slowPath)
1562     traceExecution()
1563     writeBarrierOnOperands(1, 3)
1564     loadisFromInstruction(1, t0)
1565     loadConstantOrVariableCell(t0, t1, .opPutByValSlow)
1566     loadpFromInstruction(4, t3)
1567     move t1, t2
1568     arrayProfile(t2, t3, t0)
1569     loadisFromInstruction(2, t0)
1570     loadConstantOrVariableInt32(t0, t3, .opPutByValSlow)
1571     sxi2q t3, t3
1572     loadp JSObject::m_butterfly[t1], t0
1573     andi IndexingShapeMask, t2
1574     bineq t2, Int32Shape, .opPutByValNotInt32
1575     contiguousPutByVal(
1576         macro (operand, scratch, address)
1577             loadConstantOrVariable(operand, scratch)
1578             bpb scratch, tagTypeNumber, .opPutByValSlow
1579             storep scratch, address
1580         end)
1581
1582 .opPutByValNotInt32:
1583     bineq t2, DoubleShape, .opPutByValNotDouble
1584     contiguousPutByVal(
1585         macro (operand, scratch, address)
1586             loadConstantOrVariable(operand, scratch)
1587             bqb scratch, tagTypeNumber, .notInt
1588             ci2d scratch, ft0
1589             jmp .ready
1590         .notInt:
1591             addp tagTypeNumber, scratch
1592             fq2d scratch, ft0
1593             bdnequn ft0, ft0, .opPutByValSlow
1594         .ready:
1595             stored ft0, address
1596         end)
1597
1598 .opPutByValNotDouble:
1599     bineq t2, ContiguousShape, .opPutByValNotContiguous
1600     contiguousPutByVal(
1601         macro (operand, scratch, address)
1602             loadConstantOrVariable(operand, scratch)
1603             storep scratch, address
1604         end)
1605
1606 .opPutByValNotContiguous:
1607     bineq t2, ArrayStorageShape, .opPutByValSlow
1608     biaeq t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.vectorLength[t0], .opPutByValOutOfBounds
1609     btqz ArrayStorage::m_vector[t0, t3, 8], .opPutByValArrayStorageEmpty
1610 .opPutByValArrayStorageStoreResult:
1611     loadisFromInstruction(3, t2)
1612     loadConstantOrVariable(t2, t1)
1613     storeq t1, ArrayStorage::m_vector[t0, t3, 8]
1614     dispatch(5)
1615
1616 .opPutByValArrayStorageEmpty:
1617     loadpFromInstruction(4, t1)
1618     storeb 1, ArrayProfile::m_mayStoreToHole[t1]
1619     addi 1, ArrayStorage::m_numValuesInVector[t0]
1620     bib t3, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0], .opPutByValArrayStorageStoreResult
1621     addi 1, t3, t1
1622     storei t1, -sizeof IndexingHeader + IndexingHeader::u.lengths.publicLength[t0]
1623     jmp .opPutByValArrayStorageStoreResult
1624
1625 .opPutByValOutOfBounds:
1626     loadpFromInstruction(4, t0)
1627     storeb 1, ArrayProfile::m_outOfBounds[t0]
1628 .opPutByValSlow:
1629     callSlowPath(slowPath)
1630     dispatch(5)
1631 end
1632
1633 _llint_op_put_by_val:
1634     putByVal(_llint_slow_path_put_by_val)
1635
1636 _llint_op_put_by_val_direct:
1637     putByVal(_llint_slow_path_put_by_val_direct)
1638
1639
1640 _llint_op_jmp:
1641     traceExecution()
1642     dispatchIntIndirect(1)
1643
1644
1645 macro jumpTrueOrFalse(conditionOp, slow)
1646     loadisFromInstruction(1, t1)
1647     loadConstantOrVariable(t1, t0)
1648     xorq ValueFalse, t0
1649     btqnz t0, -1, .slow
1650     conditionOp(t0, .target)
1651     dispatch(3)
1652
1653 .target:
1654     dispatchIntIndirect(2)
1655
1656 .slow:
1657     callSlowPath(slow)
1658     dispatch(0)
1659 end
1660
1661
1662 macro equalNull(cellHandler, immediateHandler)
1663     loadisFromInstruction(1, t0)
1664     assertNotConstant(t0)
1665     loadq [cfr, t0, 8], t0
1666     btqnz t0, tagMask, .immediate
1667     loadStructureWithScratch(t0, t2, t1)
1668     cellHandler(t2, JSCell::m_flags[t0], .target)
1669     dispatch(3)
1670
1671 .target:
1672     dispatchIntIndirect(2)
1673
1674 .immediate:
1675     andq ~TagBitUndefined, t0
1676     immediateHandler(t0, .target)
1677     dispatch(3)
1678 end
1679
1680 _llint_op_jeq_null:
1681     traceExecution()
1682     equalNull(
1683         macro (structure, value, target)
1684             btbz value, MasqueradesAsUndefined, .notMasqueradesAsUndefined
1685             loadp CodeBlock[cfr], t0
1686             loadp CodeBlock::m_globalObject[t0], t0
1687             bpeq Structure::m_globalObject[structure], t0, target
1688 .notMasqueradesAsUndefined:
1689         end,
1690         macro (value, target) bqeq value, ValueNull, target end)
1691
1692
1693 _llint_op_jneq_null:
1694     traceExecution()
1695     equalNull(
1696         macro (structure, value, target)
1697             btbz value, MasqueradesAsUndefined, target
1698             loadp CodeBlock[cfr], t0
1699             loadp CodeBlock::m_globalObject[t0], t0
1700             bpneq Structure::m_globalObject[structure], t0, target
1701         end,
1702         macro (value, target) bqneq value, ValueNull, target end)
1703
1704
1705 _llint_op_jneq_ptr:
1706     traceExecution()
1707     loadisFromInstruction(1, t0)
1708     loadisFromInstruction(2, t1)
1709     loadp CodeBlock[cfr], t2
1710     loadp CodeBlock::m_globalObject[t2], t2
1711     loadp JSGlobalObject::m_specialPointers[t2, t1, 8], t1
1712     bpneq t1, [cfr, t0, 8], .opJneqPtrTarget
1713     dispatch(4)
1714
1715 .opJneqPtrTarget:
1716     dispatchIntIndirect(3)
1717
1718
1719 macro compare(integerCompare, doubleCompare, slowPath)
1720     loadisFromInstruction(1, t2)
1721     loadisFromInstruction(2, t3)
1722     loadConstantOrVariable(t2, t0)
1723     loadConstantOrVariable(t3, t1)
1724     bqb t0, tagTypeNumber, .op1NotInt
1725     bqb t1, tagTypeNumber, .op2NotInt
1726     integerCompare(t0, t1, .jumpTarget)
1727     dispatch(4)
1728
1729 .op1NotInt:
1730     btqz t0, tagTypeNumber, .slow
1731     bqb t1, tagTypeNumber, .op1NotIntOp2NotInt
1732     ci2d t1, ft1
1733     jmp .op1NotIntReady
1734 .op1NotIntOp2NotInt:
1735     btqz t1, tagTypeNumber, .slow
1736     addq tagTypeNumber, t1
1737     fq2d t1, ft1
1738 .op1NotIntReady:
1739     addq tagTypeNumber, t0
1740     fq2d t0, ft0
1741     doubleCompare(ft0, ft1, .jumpTarget)
1742     dispatch(4)
1743
1744 .op2NotInt:
1745     ci2d t0, ft0
1746     btqz t1, tagTypeNumber, .slow
1747     addq tagTypeNumber, t1
1748     fq2d t1, ft1
1749     doubleCompare(ft0, ft1, .jumpTarget)
1750     dispatch(4)
1751
1752 .jumpTarget:
1753     dispatchIntIndirect(3)
1754
1755 .slow:
1756     callSlowPath(slowPath)
1757     dispatch(0)
1758 end
1759
1760
1761 _llint_op_switch_imm:
1762     traceExecution()
1763     loadisFromInstruction(3, t2)
1764     loadisFromInstruction(1, t3)
1765     loadConstantOrVariable(t2, t1)
1766     loadp CodeBlock[cfr], t2
1767     loadp CodeBlock::m_rareData[t2], t2
1768     muli sizeof SimpleJumpTable, t3    # FIXME: would be nice to peephole this!
1769     loadp CodeBlock::RareData::m_switchJumpTables + VectorBufferOffset[t2], t2
1770     addp t3, t2
1771     bqb t1, tagTypeNumber, .opSwitchImmNotInt
1772     subi SimpleJumpTable::min[t2], t1
1773     biaeq t1, SimpleJumpTable::branchOffsets + VectorSizeOffset[t2], .opSwitchImmFallThrough
1774     loadp SimpleJumpTable::branchOffsets + VectorBufferOffset[t2], t3
1775     loadis [t3, t1, 4], t1
1776     btiz t1, .opSwitchImmFallThrough
1777     dispatch(t1)
1778
1779 .opSwitchImmNotInt:
1780     btqnz t1, tagTypeNumber, .opSwitchImmSlow   # Go slow if it's a double.
1781 .opSwitchImmFallThrough:
1782     dispatchIntIndirect(2)
1783
1784 .opSwitchImmSlow:
1785     callSlowPath(_llint_slow_path_switch_imm)
1786     dispatch(0)
1787
1788
1789 _llint_op_switch_char:
1790     traceExecution()
1791     loadisFromInstruction(3, t2)
1792     loadisFromInstruction(1, t3)
1793     loadConstantOrVariable(t2, t1)
1794     loadp CodeBlock[cfr], t2
1795     loadp CodeBlock::m_rareData[t2], t2
1796     muli sizeof SimpleJumpTable, t3
1797     loadp CodeBlock::RareData::m_switchJumpTables + VectorBufferOffset[t2], t2
1798     addp t3, t2
1799     btqnz t1, tagMask, .opSwitchCharFallThrough
1800     bbneq JSCell::m_type[t1], StringType, .opSwitchCharFallThrough
1801     bineq JSString::m_length[t1], 1, .opSwitchCharFallThrough
1802     loadp JSString::m_value[t1], t0
1803     btpz  t0, .opSwitchOnRope
1804     loadp StringImpl::m_data8[t0], t1
1805     btinz StringImpl::m_hashAndFlags[t0], HashFlags8BitBuffer, .opSwitchChar8Bit
1806     loadh [t1], t0
1807     jmp .opSwitchCharReady
1808 .opSwitchChar8Bit:
1809     loadb [t1], t0
1810 .opSwitchCharReady:
1811     subi SimpleJumpTable::min[t2], t0
1812     biaeq t0, SimpleJumpTable::branchOffsets + VectorSizeOffset[t2], .opSwitchCharFallThrough
1813     loadp SimpleJumpTable::branchOffsets + VectorBufferOffset[t2], t2
1814     loadis [t2, t0, 4], t1
1815     btiz t1, .opSwitchCharFallThrough
1816     dispatch(t1)
1817
1818 .opSwitchCharFallThrough:
1819     dispatchIntIndirect(2)
1820
1821 .opSwitchOnRope:
1822     callSlowPath(_llint_slow_path_switch_char)
1823     dispatch(0)
1824
1825
1826 _llint_op_new_func:
1827     traceExecution()
1828     loadisFromInstruction(3, t2)
1829     btiz t2, .opNewFuncUnchecked
1830     loadisFromInstruction(1, t1)
1831     btqnz [cfr, t1, 8], .opNewFuncDone
1832 .opNewFuncUnchecked:
1833     callSlowPath(_llint_slow_path_new_func)
1834 .opNewFuncDone:
1835     dispatch(4)
1836
1837
1838 _llint_op_new_captured_func:
1839     traceExecution()
1840     callSlowPath(_slow_path_new_captured_func)
1841     dispatch(4)
1842
1843
1844 macro arrayProfileForCall()
1845     loadisFromInstruction(4, t3)
1846     negp t3
1847     loadq ThisArgumentOffset[cfr, t3, 8], t0
1848     btqnz t0, tagMask, .done
1849     loadpFromInstruction((CallOpCodeSize - 2), t1)
1850     loadi JSCell::m_structureID[t0], t3
1851     storei t3, ArrayProfile::m_lastSeenStructureID[t1]
1852 .done:
1853 end
1854
1855 macro doCall(slowPath)
1856     loadisFromInstruction(2, t0)
1857     loadpFromInstruction(5, t1)
1858     loadp LLIntCallLinkInfo::callee[t1], t2
1859     loadConstantOrVariable(t0, t3)
1860     bqneq t3, t2, .opCallSlow
1861     loadisFromInstruction(4, t3)
1862     lshifti 3, t3
1863     negp t3
1864     addp cfr, t3
1865     loadp JSFunction::m_scope[t2], t0
1866     storeq t2, Callee[t3]
1867     storeq t0, ScopeChain[t3]
1868     loadisFromInstruction(3, t2)
1869     storei PC, ArgumentCount + TagOffset[cfr]
1870     storei t2, ArgumentCount + PayloadOffset[t3]
1871     addp CallerFrameAndPCSize, t3
1872     callTargetFunction(t1, t3)
1873
1874 .opCallSlow:
1875     slowPathForCall(slowPath)
1876 end
1877
1878
1879 _llint_op_tear_off_activation:
1880     traceExecution()
1881     loadisFromInstruction(1, t0)
1882     btqz [cfr, t0, 8], .opTearOffActivationNotCreated
1883     callSlowPath(_llint_slow_path_tear_off_activation)
1884 .opTearOffActivationNotCreated:
1885     dispatch(2)
1886
1887
1888 _llint_op_tear_off_arguments:
1889     traceExecution()
1890     loadisFromInstruction(1, t0)
1891     addq 1, t0   # Get the unmodifiedArgumentsRegister
1892     btqz [cfr, t0, 8], .opTearOffArgumentsNotCreated
1893     callSlowPath(_llint_slow_path_tear_off_arguments)
1894 .opTearOffArgumentsNotCreated:
1895     dispatch(3)
1896
1897
1898 _llint_op_ret:
1899     traceExecution()
1900     checkSwitchToJITForEpilogue()
1901     loadisFromInstruction(1, t2)
1902     loadConstantOrVariable(t2, t0)
1903     doReturn()
1904
1905
1906 _llint_op_ret_object_or_this:
1907     traceExecution()
1908     checkSwitchToJITForEpilogue()
1909     loadisFromInstruction(1, t2)
1910     loadConstantOrVariable(t2, t0)
1911     btqnz t0, tagMask, .opRetObjectOrThisNotObject
1912     bbb JSCell::m_type[t0], ObjectType, .opRetObjectOrThisNotObject
1913     doReturn()
1914
1915 .opRetObjectOrThisNotObject:
1916     loadisFromInstruction(2, t2)
1917     loadConstantOrVariable(t2, t0)
1918     doReturn()
1919
1920
1921 _llint_op_to_primitive:
1922     traceExecution()
1923     loadisFromInstruction(2, t2)
1924     loadisFromInstruction(1, t3)
1925     loadConstantOrVariable(t2, t0)
1926     btqnz t0, tagMask, .opToPrimitiveIsImm
1927     bbneq JSCell::m_type[t0], StringType, .opToPrimitiveSlowCase
1928 .opToPrimitiveIsImm:
1929     storeq t0, [cfr, t3, 8]
1930     dispatch(3)
1931
1932 .opToPrimitiveSlowCase:
1933     callSlowPath(_slow_path_to_primitive)
1934     dispatch(3)
1935
1936
1937 _llint_op_next_pname:
1938     traceExecution()
1939     loadisFromInstruction(3, t1)
1940     loadisFromInstruction(4, t2)
1941     assertNotConstant(t1)
1942     assertNotConstant(t2)
1943     loadi PayloadOffset[cfr, t1, 8], t0
1944     bieq t0, PayloadOffset[cfr, t2, 8], .opNextPnameEnd
1945     loadisFromInstruction(5, t2)
1946     assertNotConstant(t2)
1947     loadp [cfr, t2, 8], t2
1948     loadp JSPropertyNameIterator::m_jsStrings[t2], t3
1949     loadq [t3, t0, 8], t3
1950     addi 1, t0
1951     storei t0, PayloadOffset[cfr, t1, 8]
1952     loadisFromInstruction(1, t1)
1953     storeq t3, [cfr, t1, 8]
1954     loadisFromInstruction(2, t3)
1955     assertNotConstant(t3)
1956     loadq [cfr, t3, 8], t3
1957     loadStructureWithScratch(t3, t1, t0)
1958     bpneq t1, JSPropertyNameIterator::m_cachedStructure[t2], .opNextPnameSlow
1959     loadp JSPropertyNameIterator::m_cachedPrototypeChain[t2], t0
1960     loadp StructureChain::m_vector[t0], t0
1961     btpz [t0], .opNextPnameTarget
1962 .opNextPnameCheckPrototypeLoop:
1963     bqeq Structure::m_prototype[t1], ValueNull, .opNextPnameSlow
1964     loadq Structure::m_prototype[t1], t2
1965     loadStructureWithScratch(t2, t1, t3)
1966     bpneq t1, [t0], .opNextPnameSlow
1967     addp 8, t0
1968     btpnz [t0], .opNextPnameCheckPrototypeLoop
1969 .opNextPnameTarget:
1970     dispatchIntIndirect(6)
1971
1972 .opNextPnameEnd:
1973     dispatch(7)
1974
1975 .opNextPnameSlow:
1976     callSlowPath(_llint_slow_path_next_pname) # This either keeps the PC where it was (causing us to loop) or sets it to target.
1977     dispatch(0)
1978
1979
1980 _llint_op_catch:
1981     # Gotta restore the tag registers. We could be throwing from FTL, which may
1982     # clobber them.
1983     move TagTypeNumber, tagTypeNumber
1984     move TagMask, tagMask
1985
1986     # This is where we end up from the JIT's throw trampoline (because the
1987     # machine code return address will be set to _llint_op_catch), and from
1988     # the interpreter's throw trampoline (see _llint_throw_trampoline).
1989     # The throwing code must have known that we were throwing to the interpreter,
1990     # and have set VM::targetInterpreterPCForThrow.
1991     loadp ScopeChain[cfr], t3
1992     andp MarkedBlockMask, t3
1993     loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
1994     loadp VM::callFrameForThrow[t3], cfr
1995     restoreStackPointerAfterCall()
1996
1997     loadp CodeBlock[cfr], PB
1998     loadp CodeBlock::m_instructions[PB], PB
1999     loadp VM::targetInterpreterPCForThrow[t3], PC
2000     subp PB, PC
2001     rshiftp 3, PC
2002     loadq VM::m_exception[t3], t0
2003     storeq 0, VM::m_exception[t3]
2004     loadisFromInstruction(1, t2)
2005     storeq t0, [cfr, t2, 8]
2006     traceExecution()
2007     dispatch(2)
2008
2009
2010 _llint_op_end:
2011     traceExecution()
2012     checkSwitchToJITForEpilogue()
2013     loadisFromInstruction(1, t0)
2014     assertNotConstant(t0)
2015     loadq [cfr, t0, 8], t0
2016     doReturn()
2017
2018
2019 _llint_throw_from_slow_path_trampoline:
2020     callSlowPath(_llint_slow_path_handle_exception)
2021
2022     # When throwing from the interpreter (i.e. throwing from LLIntSlowPaths), so
2023     # the throw target is not necessarily interpreted code, we come to here.
2024     # This essentially emulates the JIT's throwing protocol.
2025     loadp CodeBlock[cfr], t1
2026     loadp CodeBlock::m_vm[t1], t1
2027     jmp VM::targetMachinePCForThrow[t1]
2028
2029
2030 _llint_throw_during_call_trampoline:
2031     preserveReturnAddressAfterCall(t2)
2032     jmp _llint_throw_from_slow_path_trampoline
2033
2034
2035 macro nativeCallTrampoline(executableOffsetToFunction)
2036
2037     functionPrologue()
2038     storep 0, CodeBlock[cfr]
2039     if X86_64 or X86_64_WIN
2040         if X86_64
2041             const arg1 = t4  # t4 = rdi
2042             const arg2 = t5  # t5 = rsi
2043             const temp = t1
2044         elsif X86_64_WIN
2045             const arg1 = t2  # t2 = rcx
2046             const arg2 = t1  # t1 = rdx
2047             const temp = t0
2048         end
2049         loadp ScopeChain[cfr], t0
2050         andp MarkedBlockMask, t0
2051         loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t0], t0
2052         storep cfr, VM::topCallFrame[t0]
2053         loadp CallerFrame[cfr], t0
2054         loadq ScopeChain[t0], t1
2055         storeq t1, ScopeChain[cfr]
2056         move cfr, arg1
2057         loadp Callee[cfr], arg2
2058         loadp JSFunction::m_executable[arg2], temp
2059         checkStackPointerAlignment(t3, 0xdead0001)
2060         if X86_64_WIN
2061             subp 32, sp
2062         end
2063         call executableOffsetToFunction[temp]
2064         if X86_64_WIN
2065             addp 32, sp
2066         end
2067         loadp ScopeChain[cfr], t3
2068         andp MarkedBlockMask, t3
2069         loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
2070     elsif ARM64 or C_LOOP
2071         loadp ScopeChain[cfr], t0
2072         andp MarkedBlockMask, t0
2073         loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t0], t0
2074         storep cfr, VM::topCallFrame[t0]
2075         loadp CallerFrame[cfr], t2
2076         loadp ScopeChain[t2], t1
2077         storep t1, ScopeChain[cfr]
2078         preserveReturnAddressAfterCall(t3)
2079         storep t3, ReturnPC[cfr]
2080         move cfr, t0
2081         loadp Callee[cfr], t1
2082         loadp JSFunction::m_executable[t1], t1
2083         move t2, cfr # Restore cfr to avoid loading from stack
2084         if C_LOOP
2085             cloopCallNative executableOffsetToFunction[t1]
2086         else
2087             call executableOffsetToFunction[t1]
2088         end
2089         restoreReturnAddressBeforeReturn(t3)
2090         loadp ScopeChain[cfr], t3
2091         andp MarkedBlockMask, t3
2092         loadp MarkedBlock::m_weakSet + WeakSet::m_vm[t3], t3
2093     else
2094         error
2095     end
2096
2097     functionEpilogue()
2098
2099     btqnz VM::m_exception[t3], .handleException
2100     ret
2101
2102 .handleException:
2103     storep cfr, VM::topCallFrame[t3]
2104     restoreStackPointerAfterCall()
2105     jmp _llint_throw_from_slow_path_trampoline
2106 end
2107
2108
2109 macro getGlobalObject(dst)
2110     loadp CodeBlock[cfr], t0
2111     loadp CodeBlock::m_globalObject[t0], t0
2112     loadisFromInstruction(dst, t1)
2113     storeq t0, [cfr, t1, 8]
2114 end
2115
2116 macro varInjectionCheck(slowPath)
2117     loadp CodeBlock[cfr], t0
2118     loadp CodeBlock::m_globalObject[t0], t0
2119     loadp JSGlobalObject::m_varInjectionWatchpoint[t0], t0
2120     bbeq WatchpointSet::m_state[t0], IsInvalidated, slowPath
2121 end
2122
2123 macro resolveScope()
2124     loadp CodeBlock[cfr], t0
2125     loadisFromInstruction(4, t2)
2126     btbz CodeBlock::m_needsActivation[t0], .resolveScopeAfterActivationCheck
2127     loadis CodeBlock::m_activationRegister[t0], t1
2128     btpz [cfr, t1, 8], .resolveScopeAfterActivationCheck
2129     addi 1, t2
2130
2131 .resolveScopeAfterActivationCheck:
2132     loadp ScopeChain[cfr], t0
2133     btiz t2, .resolveScopeLoopEnd
2134
2135 .resolveScopeLoop:
2136     loadp JSScope::m_next[t0], t0
2137     subi 1, t2
2138     btinz t2, .resolveScopeLoop
2139
2140 .resolveScopeLoopEnd:
2141     loadisFromInstruction(1, t1)
2142     storeq t0, [cfr, t1, 8]
2143 end
2144
2145
2146 _llint_op_resolve_scope:
2147     traceExecution()
2148     loadisFromInstruction(3, t0)
2149
2150 #rGlobalProperty:
2151     bineq t0, GlobalProperty, .rGlobalVar
2152     getGlobalObject(1)
2153     dispatch(6)
2154
2155 .rGlobalVar:
2156     bineq t0, GlobalVar, .rClosureVar
2157     getGlobalObject(1)
2158     dispatch(6)
2159
2160 .rClosureVar:
2161     bineq t0, ClosureVar, .rGlobalPropertyWithVarInjectionChecks
2162     resolveScope()
2163     dispatch(6)
2164
2165 .rGlobalPropertyWithVarInjectionChecks:
2166     bineq t0, GlobalPropertyWithVarInjectionChecks, .rGlobalVarWithVarInjectionChecks
2167     varInjectionCheck(.rDynamic)
2168     getGlobalObject(1)
2169     dispatch(6)
2170
2171 .rGlobalVarWithVarInjectionChecks:
2172     bineq t0, GlobalVarWithVarInjectionChecks, .rClosureVarWithVarInjectionChecks
2173     varInjectionCheck(.rDynamic)
2174     getGlobalObject(1)
2175     dispatch(6)
2176
2177 .rClosureVarWithVarInjectionChecks:
2178     bineq t0, ClosureVarWithVarInjectionChecks, .rDynamic
2179     varInjectionCheck(.rDynamic)
2180     resolveScope()
2181     dispatch(6)
2182
2183 .rDynamic:
2184     callSlowPath(_llint_slow_path_resolve_scope)
2185     dispatch(6)
2186
2187
2188 macro loadWithStructureCheck(operand, slowPath)
2189     loadisFromInstruction(operand, t0)
2190     loadq [cfr, t0, 8], t0
2191     loadStructureWithScratch(t0, t2, t1)
2192     loadpFromInstruction(5, t1)
2193     bpneq t2, t1, slowPath
2194 end
2195
2196 macro getProperty()
2197     loadisFromInstruction(6, t1)
2198     loadPropertyAtVariableOffset(t1, t0, t2)
2199     valueProfile(t2, 7, t0)
2200     loadisFromInstruction(1, t0)
2201     storeq t2, [cfr, t0, 8]
2202 end
2203
2204 macro getGlobalVar()
2205     loadpFromInstruction(6, t0)
2206     loadq [t0], t0
2207     valueProfile(t0, 7, t1)
2208     loadisFromInstruction(1, t1)
2209     storeq t0, [cfr, t1, 8]
2210 end
2211
2212 macro getClosureVar()
2213     loadp JSVariableObject::m_registers[t0], t0
2214     loadisFromInstruction(6, t1)
2215     loadq [t0, t1, 8], t0
2216     valueProfile(t0, 7, t1)
2217     loadisFromInstruction(1, t1)
2218     storeq t0, [cfr, t1, 8]
2219 end
2220
2221 _llint_op_get_from_scope:
2222     traceExecution()
2223     loadisFromInstruction(4, t0)
2224     andi ResolveModeMask, t0
2225
2226 #gGlobalProperty:
2227     bineq t0, GlobalProperty, .gGlobalVar
2228     loadWithStructureCheck(2, .gDynamic)
2229     getProperty()
2230     dispatch(8)
2231
2232 .gGlobalVar:
2233     bineq t0, GlobalVar, .gClosureVar
2234     getGlobalVar()
2235     dispatch(8)
2236
2237 .gClosureVar:
2238     bineq t0, ClosureVar, .gGlobalPropertyWithVarInjectionChecks
2239     loadVariable(2, t0)
2240     getClosureVar()
2241     dispatch(8)
2242
2243 .gGlobalPropertyWithVarInjectionChecks:
2244     bineq t0, GlobalPropertyWithVarInjectionChecks, .gGlobalVarWithVarInjectionChecks
2245     loadWithStructureCheck(2, .gDynamic)
2246     getProperty()
2247     dispatch(8)
2248
2249 .gGlobalVarWithVarInjectionChecks:
2250     bineq t0, GlobalVarWithVarInjectionChecks, .gClosureVarWithVarInjectionChecks
2251     varInjectionCheck(.gDynamic)
2252     loadVariable(2, t0)
2253     getGlobalVar()
2254     dispatch(8)
2255
2256 .gClosureVarWithVarInjectionChecks:
2257     bineq t0, ClosureVarWithVarInjectionChecks, .gDynamic
2258     varInjectionCheck(.gDynamic)
2259     loadVariable(2, t0)
2260     getClosureVar()
2261     dispatch(8)
2262
2263 .gDynamic:
2264     callSlowPath(_llint_slow_path_get_from_scope)
2265     dispatch(8)
2266
2267
2268 macro putProperty()
2269     loadisFromInstruction(3, t1)
2270     loadConstantOrVariable(t1, t2)
2271     loadisFromInstruction(6, t1)
2272     storePropertyAtVariableOffset(t1, t0, t2)
2273 end
2274
2275 macro putGlobalVar()
2276     loadisFromInstruction(3, t0)
2277     loadConstantOrVariable(t0, t1)
2278     loadpFromInstruction(5, t2)
2279     notifyWrite(t2, t1, t0, .pDynamic)
2280     loadpFromInstruction(6, t0)
2281     storeq t1, [t0]
2282 end
2283
2284 macro putClosureVar()
2285     loadisFromInstruction(3, t1)
2286     loadConstantOrVariable(t1, t2)
2287     loadp JSVariableObject::m_registers[t0], t0
2288     loadisFromInstruction(6, t1)
2289     storeq t2, [t0, t1, 8]
2290 end
2291
2292
2293 _llint_op_put_to_scope:
2294     traceExecution()
2295     loadisFromInstruction(4, t0)
2296     andi ResolveModeMask, t0
2297
2298 #pGlobalProperty:
2299     bineq t0, GlobalProperty, .pGlobalVar
2300     writeBarrierOnOperands(1, 3)
2301     loadWithStructureCheck(1, .pDynamic)
2302     putProperty()
2303     dispatch(7)
2304
2305 .pGlobalVar:
2306     bineq t0, GlobalVar, .pClosureVar
2307     writeBarrierOnGlobalObject(3)
2308     putGlobalVar()
2309     dispatch(7)
2310
2311 .pClosureVar:
2312     bineq t0, ClosureVar, .pGlobalPropertyWithVarInjectionChecks
2313     writeBarrierOnOperands(1, 3)
2314     loadVariable(1, t0)
2315     putClosureVar()
2316     dispatch(7)
2317
2318 .pGlobalPropertyWithVarInjectionChecks:
2319     bineq t0, GlobalPropertyWithVarInjectionChecks, .pGlobalVarWithVarInjectionChecks
2320     writeBarrierOnOperands(1, 3)
2321     loadWithStructureCheck(1, .pDynamic)
2322     putProperty()
2323     dispatch(7)
2324
2325 .pGlobalVarWithVarInjectionChecks:
2326     bineq t0, GlobalVarWithVarInjectionChecks, .pClosureVarWithVarInjectionChecks
2327     writeBarrierOnGlobalObject(3)
2328     varInjectionCheck(.pDynamic)
2329     putGlobalVar()
2330     dispatch(7)
2331
2332 .pClosureVarWithVarInjectionChecks:
2333     bineq t0, ClosureVarWithVarInjectionChecks, .pDynamic
2334     writeBarrierOnOperands(1, 3)
2335     varInjectionCheck(.pDynamic)
2336     loadVariable(1, t0)
2337     putClosureVar()
2338     dispatch(7)
2339
2340 .pDynamic:
2341     callSlowPath(_llint_slow_path_put_to_scope)
2342     dispatch(7)