]> git.saurik.com Git - apple/javascriptcore.git/blame_incremental - ftl/FTLLowerDFGToLLVM.cpp
projects / apple / javascriptcore.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
JavaScriptCore-7601.1.46.3.tar.gz
[apple/javascriptcore.git] / ftl / FTLLowerDFGToLLVM.cpp
... / ...
CommitLineData
1/*
2 * Copyright (C) 2013-2015 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "FTLLowerDFGToLLVM.h"
28
29#if ENABLE(FTL_JIT)
30
31#include "CodeBlockWithJITType.h"
32#include "DFGAbstractInterpreterInlines.h"
33#include "DFGInPlaceAbstractState.h"
34#include "DFGOSRAvailabilityAnalysisPhase.h"
35#include "DFGOSRExitFuzz.h"
36#include "DirectArguments.h"
37#include "FTLAbstractHeapRepository.h"
38#include "FTLAvailableRecovery.h"
39#include "FTLForOSREntryJITCode.h"
40#include "FTLFormattedValue.h"
41#include "FTLInlineCacheSize.h"
42#include "FTLLoweredNodeValue.h"
43#include "FTLOperations.h"
44#include "FTLOutput.h"
45#include "FTLThunks.h"
46#include "FTLWeightedTarget.h"
47#include "JSCInlines.h"
48#include "JSLexicalEnvironment.h"
49#include "OperandsInlines.h"
50#include "ScopedArguments.h"
51#include "ScopedArgumentsTable.h"
52#include "VirtualRegister.h"
53#include <atomic>
54#include <dlfcn.h>
55#include <llvm/InitializeLLVM.h>
56#include <unordered_set>
57#include <wtf/ProcessID.h>
58
59#if ENABLE(FTL_NATIVE_CALL_INLINING)
60#include "BundlePath.h"
61#endif
62
63namespace JSC { namespace FTL {
64
65using namespace DFG;
66
67namespace {
68
69std::atomic<int> compileCounter;
70
71#if ASSERT_DISABLED
72NO_RETURN_DUE_TO_CRASH static void ftlUnreachable()
73{
74 CRASH();
75}
76#else
77NO_RETURN_DUE_TO_CRASH static void ftlUnreachable(
78 CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex)
79{
80 dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex);
81 if (nodeIndex != UINT_MAX)
82 dataLog(", node @", nodeIndex);
83 dataLog(".\n");
84 CRASH();
85}
86#endif
87
88// Using this instead of typeCheck() helps to reduce the load on LLVM, by creating
89// significantly less dead code.
90#define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) do { \
91 FormattedValue _ftc_lowValue = (lowValue); \
92 Edge _ftc_highValue = (highValue); \
93 SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough); \
94 if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \
95 break; \
96 typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition)); \
97 } while (false)
98
99class LowerDFGToLLVM {
100public:
101 LowerDFGToLLVM(State& state)
102 : m_graph(state.graph)
103 , m_ftlState(state)
104 , m_heaps(state.context)
105 , m_out(state.context)
106 , m_state(state.graph)
107 , m_interpreter(state.graph, m_state)
108 , m_stackmapIDs(0)
109 , m_tbaaKind(mdKindID(state.context, "tbaa"))
110 , m_tbaaStructKind(mdKindID(state.context, "tbaa.struct"))
111 {
112 }
113
114 void lower()
115 {
116 CString name;
117 if (verboseCompilationEnabled()) {
118 name = toCString(
119 "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(),
120 "_", codeBlock()->hash());
121 } else
122 name = "jsBody";
123
124 m_graph.m_dominators.computeIfNecessary(m_graph);
125
126 m_ftlState.module =
127 moduleCreateWithNameInContext(name.data(), m_ftlState.context);
128
129 m_ftlState.function = addFunction(
130 m_ftlState.module, name.data(), functionType(m_out.int64));
131 setFunctionCallingConv(m_ftlState.function, LLVMCCallConv);
132 if (isX86() && Options::llvmDisallowAVX()) {
133 // AVX makes V8/raytrace 80% slower. It makes Kraken/audio-oscillator 4.5x
134 // slower. It should be disabled.
135 addTargetDependentFunctionAttr(m_ftlState.function, "target-features", "-avx");
136 }
137
138 if (verboseCompilationEnabled())
139 dataLog("Function ready, beginning lowering.\n");
140
141 m_out.initialize(m_ftlState.module, m_ftlState.function, m_heaps);
142
143 m_prologue = FTL_NEW_BLOCK(m_out, ("Prologue"));
144 LBasicBlock stackOverflow = FTL_NEW_BLOCK(m_out, ("Stack overflow"));
145 m_handleExceptions = FTL_NEW_BLOCK(m_out, ("Handle Exceptions"));
146
147 LBasicBlock checkArguments = FTL_NEW_BLOCK(m_out, ("Check arguments"));
148
149 for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
150 m_highBlock = m_graph.block(blockIndex);
151 if (!m_highBlock)
152 continue;
153 m_blocks.add(m_highBlock, FTL_NEW_BLOCK(m_out, ("Block ", *m_highBlock)));
154 }
155
156 m_out.appendTo(m_prologue, stackOverflow);
157 createPhiVariables();
158
159 auto preOrder = m_graph.blocksInPreOrder();
160
161 int maxNumberOfArguments = -1;
162 for (BasicBlock* block : preOrder) {
163 for (unsigned nodeIndex = block->size(); nodeIndex--; ) {
164 Node* node = block->at(nodeIndex);
165 switch (node->op()) {
166 case NativeCall:
167 case NativeConstruct: {
168 int numArgs = node->numChildren();
169 if (numArgs > maxNumberOfArguments)
170 maxNumberOfArguments = numArgs;
171 break;
172 }
173 default:
174 break;
175 }
176 }
177 }
178
179 if (maxNumberOfArguments >= 0) {
180 m_execState = m_out.alloca(arrayType(m_out.int64, JSStack::CallFrameHeaderSize + maxNumberOfArguments));
181 m_execStorage = m_out.ptrToInt(m_execState, m_out.intPtr);
182 }
183
184 LValue capturedAlloca = m_out.alloca(arrayType(m_out.int64, m_graph.m_nextMachineLocal));
185
186 m_captured = m_out.add(
187 m_out.ptrToInt(capturedAlloca, m_out.intPtr),
188 m_out.constIntPtr(m_graph.m_nextMachineLocal * sizeof(Register)));
189
190 m_ftlState.capturedStackmapID = m_stackmapIDs++;
191 m_out.call(
192 m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.capturedStackmapID),
193 m_out.int32Zero, capturedAlloca);
194
195 // If we have any CallVarargs then we nee to have a spill slot for it.
196 bool hasVarargs = false;
197 for (BasicBlock* block : preOrder) {
198 for (Node* node : *block) {
199 switch (node->op()) {
200 case CallVarargs:
201 case CallForwardVarargs:
202 case ConstructVarargs:
203 case ConstructForwardVarargs:
204 hasVarargs = true;
205 break;
206 default:
207 break;
208 }
209 }
210 }
211 if (hasVarargs) {
212 LValue varargsSpillSlots = m_out.alloca(
213 arrayType(m_out.int64, JSCallVarargs::numSpillSlotsNeeded()));
214 m_ftlState.varargsSpillSlotsStackmapID = m_stackmapIDs++;
215 m_out.call(
216 m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.varargsSpillSlotsStackmapID),
217 m_out.int32Zero, varargsSpillSlots);
218 }
219
220 // We should not create any alloca's after this point, since they will cease to
221 // be mem2reg candidates.
222
223 m_callFrame = m_out.ptrToInt(
224 m_out.call(m_out.frameAddressIntrinsic(), m_out.int32Zero), m_out.intPtr);
225 m_tagTypeNumber = m_out.constInt64(TagTypeNumber);
226 m_tagMask = m_out.constInt64(TagMask);
227
228 m_out.storePtr(m_out.constIntPtr(codeBlock()), addressFor(JSStack::CodeBlock));
229
230 m_out.branch(
231 didOverflowStack(), rarely(stackOverflow), usually(checkArguments));
232
233 m_out.appendTo(stackOverflow, m_handleExceptions);
234 m_out.call(m_out.operation(operationThrowStackOverflowError), m_callFrame, m_out.constIntPtr(codeBlock()));
235 m_ftlState.handleStackOverflowExceptionStackmapID = m_stackmapIDs++;
236 m_out.call(
237 m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleStackOverflowExceptionStackmapID),
238 m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
239 m_out.unreachable();
240
241 m_out.appendTo(m_handleExceptions, checkArguments);
242 m_ftlState.handleExceptionStackmapID = m_stackmapIDs++;
243 m_out.call(
244 m_out.stackmapIntrinsic(), m_out.constInt64(m_ftlState.handleExceptionStackmapID),
245 m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
246 m_out.unreachable();
247
248 m_out.appendTo(checkArguments, lowBlock(m_graph.block(0)));
249 availabilityMap().clear();
250 availabilityMap().m_locals = Operands<Availability>(codeBlock()->numParameters(), 0);
251 for (unsigned i = codeBlock()->numParameters(); i--;) {
252 availabilityMap().m_locals.argument(i) =
253 Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i)));
254 }
255 m_codeOriginForExitTarget = CodeOrigin(0);
256 m_codeOriginForExitProfile = CodeOrigin(0);
257 m_node = nullptr;
258 for (unsigned i = codeBlock()->numParameters(); i--;) {
259 Node* node = m_graph.m_arguments[i];
260 VirtualRegister operand = virtualRegisterForArgument(i);
261
262 LValue jsValue = m_out.load64(addressFor(operand));
263
264 if (node) {
265 DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal);
266
267 // This is a hack, but it's an effective one. It allows us to do CSE on the
268 // primordial load of arguments. This assumes that the GetLocal that got put in
269 // place of the original SetArgument doesn't have any effects before it. This
270 // should hold true.
271 m_loadedArgumentValues.add(node, jsValue);
272 }
273
274 switch (m_graph.m_argumentFormats[i]) {
275 case FlushedInt32:
276 speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue));
277 break;
278 case FlushedBoolean:
279 speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue));
280 break;
281 case FlushedCell:
282 speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue));
283 break;
284 case FlushedJSValue:
285 break;
286 default:
287 DFG_CRASH(m_graph, node, "Bad flush format for argument");
288 break;
289 }
290 }
291 m_out.jump(lowBlock(m_graph.block(0)));
292
293 for (BasicBlock* block : preOrder)
294 compileBlock(block);
295
296 if (Options::dumpLLVMIR())
297 dumpModule(m_ftlState.module);
298
299 if (verboseCompilationEnabled())
300 m_ftlState.dumpState("after lowering");
301 if (validationEnabled())
302 verifyModule(m_ftlState.module);
303 }
304
305private:
306
307 void createPhiVariables()
308 {
309 for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
310 BasicBlock* block = m_graph.block(blockIndex);
311 if (!block)
312 continue;
313 for (unsigned nodeIndex = block->size(); nodeIndex--;) {
314 Node* node = block->at(nodeIndex);
315 if (node->op() != Phi)
316 continue;
317 LType type;
318 switch (node->flags() & NodeResultMask) {
319 case NodeResultDouble:
320 type = m_out.doubleType;
321 break;
322 case NodeResultInt32:
323 type = m_out.int32;
324 break;
325 case NodeResultInt52:
326 type = m_out.int64;
327 break;
328 case NodeResultBoolean:
329 type = m_out.boolean;
330 break;
331 case NodeResultJS:
332 type = m_out.int64;
333 break;
334 default:
335 DFG_CRASH(m_graph, node, "Bad Phi node result type");
336 break;
337 }
338 m_phis.add(node, buildAlloca(m_out.m_builder, type));
339 }
340 }
341 }
342
343 void compileBlock(BasicBlock* block)
344 {
345 if (!block)
346 return;
347
348 if (verboseCompilationEnabled())
349 dataLog("Compiling block ", *block, "\n");
350
351 m_highBlock = block;
352
353 LBasicBlock lowBlock = m_blocks.get(m_highBlock);
354
355 m_nextHighBlock = 0;
356 for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) {
357 m_nextHighBlock = m_graph.block(nextBlockIndex);
358 if (m_nextHighBlock)
359 break;
360 }
361 m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0;
362
363 // All of this effort to find the next block gives us the ability to keep the
364 // generated IR in roughly program order. This ought not affect the performance
365 // of the generated code (since we expect LLVM to reorder things) but it will
366 // make IR dumps easier to read.
367 m_out.appendTo(lowBlock, m_nextLowBlock);
368
369 if (Options::ftlCrashes())
370 m_out.trap();
371
372 if (!m_highBlock->cfaHasVisited) {
373 if (verboseCompilationEnabled())
374 dataLog("Bailing because CFA didn't reach.\n");
375 crash(m_highBlock->index, UINT_MAX);
376 return;
377 }
378
379 m_availabilityCalculator.beginBlock(m_highBlock);
380
381 m_state.reset();
382 m_state.beginBasicBlock(m_highBlock);
383
384 for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) {
385 if (!compileNode(m_nodeIndex))
386 break;
387 }
388 }
389
390 void safelyInvalidateAfterTermination()
391 {
392 if (verboseCompilationEnabled())
393 dataLog("Bailing.\n");
394 crash();
395
396 // Invalidate dominated blocks. Under normal circumstances we would expect
397 // them to be invalidated already. But you can have the CFA become more
398 // precise over time because the structures of objects change on the main
399 // thread. Failing to do this would result in weird crashes due to a value
400 // being used but not defined. Race conditions FTW!
401 for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
402 BasicBlock* target = m_graph.block(blockIndex);
403 if (!target)
404 continue;
405 if (m_graph.m_dominators.dominates(m_highBlock, target)) {
406 if (verboseCompilationEnabled())
407 dataLog("Block ", *target, " will bail also.\n");
408 target->cfaHasVisited = false;
409 }
410 }
411 }
412
413 bool compileNode(unsigned nodeIndex)
414 {
415 if (!m_state.isValid()) {
416 safelyInvalidateAfterTermination();
417 return false;
418 }
419
420 m_node = m_highBlock->at(nodeIndex);
421 m_codeOriginForExitProfile = m_node->origin.semantic;
422 m_codeOriginForExitTarget = m_node->origin.forExit;
423
424 if (verboseCompilationEnabled())
425 dataLog("Lowering ", m_node, "\n");
426
427 m_availableRecoveries.resize(0);
428
429 m_interpreter.startExecuting();
430
431 switch (m_node->op()) {
432 case Upsilon:
433 compileUpsilon();
434 break;
435 case Phi:
436 compilePhi();
437 break;
438 case JSConstant:
439 break;
440 case DoubleConstant:
441 compileDoubleConstant();
442 break;
443 case Int52Constant:
444 compileInt52Constant();
445 break;
446 case DoubleRep:
447 compileDoubleRep();
448 break;
449 case DoubleAsInt32:
450 compileDoubleAsInt32();
451 break;
452 case ValueRep:
453 compileValueRep();
454 break;
455 case Int52Rep:
456 compileInt52Rep();
457 break;
458 case ValueToInt32:
459 compileValueToInt32();
460 break;
461 case BooleanToNumber:
462 compileBooleanToNumber();
463 break;
464 case ExtractOSREntryLocal:
465 compileExtractOSREntryLocal();
466 break;
467 case GetStack:
468 compileGetStack();
469 break;
470 case PutStack:
471 compilePutStack();
472 break;
473 case Check:
474 compileNoOp();
475 break;
476 case ToThis:
477 compileToThis();
478 break;
479 case ValueAdd:
480 compileValueAdd();
481 break;
482 case ArithAdd:
483 case ArithSub:
484 compileArithAddOrSub();
485 break;
486 case ArithClz32:
487 compileArithClz32();
488 break;
489 case ArithMul:
490 compileArithMul();
491 break;
492 case ArithDiv:
493 compileArithDiv();
494 break;
495 case ArithMod:
496 compileArithMod();
497 break;
498 case ArithMin:
499 case ArithMax:
500 compileArithMinOrMax();
501 break;
502 case ArithAbs:
503 compileArithAbs();
504 break;
505 case ArithSin:
506 compileArithSin();
507 break;
508 case ArithCos:
509 compileArithCos();
510 break;
511 case ArithPow:
512 compileArithPow();
513 break;
514 case ArithRound:
515 compileArithRound();
516 break;
517 case ArithSqrt:
518 compileArithSqrt();
519 break;
520 case ArithLog:
521 compileArithLog();
522 break;
523 case ArithFRound:
524 compileArithFRound();
525 break;
526 case ArithNegate:
527 compileArithNegate();
528 break;
529 case BitAnd:
530 compileBitAnd();
531 break;
532 case BitOr:
533 compileBitOr();
534 break;
535 case BitXor:
536 compileBitXor();
537 break;
538 case BitRShift:
539 compileBitRShift();
540 break;
541 case BitLShift:
542 compileBitLShift();
543 break;
544 case BitURShift:
545 compileBitURShift();
546 break;
547 case UInt32ToNumber:
548 compileUInt32ToNumber();
549 break;
550 case CheckStructure:
551 compileCheckStructure();
552 break;
553 case CheckCell:
554 compileCheckCell();
555 break;
556 case CheckNotEmpty:
557 compileCheckNotEmpty();
558 break;
559 case CheckBadCell:
560 compileCheckBadCell();
561 break;
562 case GetExecutable:
563 compileGetExecutable();
564 break;
565 case ArrayifyToStructure:
566 compileArrayifyToStructure();
567 break;
568 case PutStructure:
569 compilePutStructure();
570 break;
571 case GetById:
572 compileGetById();
573 break;
574 case In:
575 compileIn();
576 break;
577 case PutById:
578 case PutByIdDirect:
579 compilePutById();
580 break;
581 case GetButterfly:
582 compileGetButterfly();
583 break;
584 case ConstantStoragePointer:
585 compileConstantStoragePointer();
586 break;
587 case GetIndexedPropertyStorage:
588 compileGetIndexedPropertyStorage();
589 break;
590 case CheckArray:
591 compileCheckArray();
592 break;
593 case GetArrayLength:
594 compileGetArrayLength();
595 break;
596 case CheckInBounds:
597 compileCheckInBounds();
598 break;
599 case GetByVal:
600 compileGetByVal();
601 break;
602 case GetMyArgumentByVal:
603 compileGetMyArgumentByVal();
604 break;
605 case PutByVal:
606 case PutByValAlias:
607 case PutByValDirect:
608 compilePutByVal();
609 break;
610 case ArrayPush:
611 compileArrayPush();
612 break;
613 case ArrayPop:
614 compileArrayPop();
615 break;
616 case CreateActivation:
617 compileCreateActivation();
618 break;
619 case NewFunction:
620 compileNewFunction();
621 break;
622 case CreateDirectArguments:
623 compileCreateDirectArguments();
624 break;
625 case CreateScopedArguments:
626 compileCreateScopedArguments();
627 break;
628 case CreateClonedArguments:
629 compileCreateClonedArguments();
630 break;
631 case NewObject:
632 compileNewObject();
633 break;
634 case NewArray:
635 compileNewArray();
636 break;
637 case NewArrayBuffer:
638 compileNewArrayBuffer();
639 break;
640 case NewArrayWithSize:
641 compileNewArrayWithSize();
642 break;
643 case GetTypedArrayByteOffset:
644 compileGetTypedArrayByteOffset();
645 break;
646 case AllocatePropertyStorage:
647 compileAllocatePropertyStorage();
648 break;
649 case ReallocatePropertyStorage:
650 compileReallocatePropertyStorage();
651 break;
652 case ToString:
653 case CallStringConstructor:
654 compileToStringOrCallStringConstructor();
655 break;
656 case ToPrimitive:
657 compileToPrimitive();
658 break;
659 case MakeRope:
660 compileMakeRope();
661 break;
662 case StringCharAt:
663 compileStringCharAt();
664 break;
665 case StringCharCodeAt:
666 compileStringCharCodeAt();
667 break;
668 case GetByOffset:
669 case GetGetterSetterByOffset:
670 compileGetByOffset();
671 break;
672 case GetGetter:
673 compileGetGetter();
674 break;
675 case GetSetter:
676 compileGetSetter();
677 break;
678 case MultiGetByOffset:
679 compileMultiGetByOffset();
680 break;
681 case PutByOffset:
682 compilePutByOffset();
683 break;
684 case MultiPutByOffset:
685 compileMultiPutByOffset();
686 break;
687 case GetGlobalVar:
688 compileGetGlobalVar();
689 break;
690 case PutGlobalVar:
691 compilePutGlobalVar();
692 break;
693 case NotifyWrite:
694 compileNotifyWrite();
695 break;
696 case GetCallee:
697 compileGetCallee();
698 break;
699 case GetArgumentCount:
700 compileGetArgumentCount();
701 break;
702 case GetScope:
703 compileGetScope();
704 break;
705 case SkipScope:
706 compileSkipScope();
707 break;
708 case GetClosureVar:
709 compileGetClosureVar();
710 break;
711 case PutClosureVar:
712 compilePutClosureVar();
713 break;
714 case GetFromArguments:
715 compileGetFromArguments();
716 break;
717 case PutToArguments:
718 compilePutToArguments();
719 break;
720 case CompareEq:
721 compileCompareEq();
722 break;
723 case CompareEqConstant:
724 compileCompareEqConstant();
725 break;
726 case CompareStrictEq:
727 compileCompareStrictEq();
728 break;
729 case CompareLess:
730 compileCompareLess();
731 break;
732 case CompareLessEq:
733 compileCompareLessEq();
734 break;
735 case CompareGreater:
736 compileCompareGreater();
737 break;
738 case CompareGreaterEq:
739 compileCompareGreaterEq();
740 break;
741 case LogicalNot:
742 compileLogicalNot();
743 break;
744 case Call:
745 case Construct:
746 compileCallOrConstruct();
747 break;
748 case CallVarargs:
749 case CallForwardVarargs:
750 case ConstructVarargs:
751 case ConstructForwardVarargs:
752 compileCallOrConstructVarargs();
753 break;
754 case LoadVarargs:
755 compileLoadVarargs();
756 break;
757 case ForwardVarargs:
758 compileForwardVarargs();
759 break;
760#if ENABLE(FTL_NATIVE_CALL_INLINING)
761 case NativeCall:
762 case NativeConstruct:
763 compileNativeCallOrConstruct();
764 break;
765#endif
766 case Jump:
767 compileJump();
768 break;
769 case Branch:
770 compileBranch();
771 break;
772 case Switch:
773 compileSwitch();
774 break;
775 case Return:
776 compileReturn();
777 break;
778 case ForceOSRExit:
779 compileForceOSRExit();
780 break;
781 case Throw:
782 case ThrowReferenceError:
783 compileThrow();
784 break;
785 case InvalidationPoint:
786 compileInvalidationPoint();
787 break;
788 case IsUndefined:
789 compileIsUndefined();
790 break;
791 case IsBoolean:
792 compileIsBoolean();
793 break;
794 case IsNumber:
795 compileIsNumber();
796 break;
797 case IsString:
798 compileIsString();
799 break;
800 case IsObject:
801 compileIsObject();
802 break;
803 case IsObjectOrNull:
804 compileIsObjectOrNull();
805 break;
806 case IsFunction:
807 compileIsFunction();
808 break;
809 case TypeOf:
810 compileTypeOf();
811 break;
812 case CheckHasInstance:
813 compileCheckHasInstance();
814 break;
815 case InstanceOf:
816 compileInstanceOf();
817 break;
818 case CountExecution:
819 compileCountExecution();
820 break;
821 case StoreBarrier:
822 compileStoreBarrier();
823 break;
824 case HasIndexedProperty:
825 compileHasIndexedProperty();
826 break;
827 case HasGenericProperty:
828 compileHasGenericProperty();
829 break;
830 case HasStructureProperty:
831 compileHasStructureProperty();
832 break;
833 case GetDirectPname:
834 compileGetDirectPname();
835 break;
836 case GetEnumerableLength:
837 compileGetEnumerableLength();
838 break;
839 case GetPropertyEnumerator:
840 compileGetPropertyEnumerator();
841 break;
842 case GetEnumeratorStructurePname:
843 compileGetEnumeratorStructurePname();
844 break;
845 case GetEnumeratorGenericPname:
846 compileGetEnumeratorGenericPname();
847 break;
848 case ToIndexString:
849 compileToIndexString();
850 break;
851 case CheckStructureImmediate:
852 compileCheckStructureImmediate();
853 break;
854 case MaterializeNewObject:
855 compileMaterializeNewObject();
856 break;
857 case MaterializeCreateActivation:
858 compileMaterializeCreateActivation();
859 break;
860
861 case PhantomLocal:
862 case LoopHint:
863 case MovHint:
864 case ZombieHint:
865 case PhantomNewObject:
866 case PhantomNewFunction:
867 case PhantomCreateActivation:
868 case PhantomDirectArguments:
869 case PhantomClonedArguments:
870 case PutHint:
871 case BottomValue:
872 case KillStack:
873 break;
874 default:
875 DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend");
876 break;
877 }
878
879 if (m_node->isTerminal())
880 return false;
881
882 if (!m_state.isValid()) {
883 safelyInvalidateAfterTermination();
884 return false;
885 }
886
887 m_availabilityCalculator.executeNode(m_node);
888 m_interpreter.executeEffects(nodeIndex);
889
890 return true;
891 }
892
893 void compileUpsilon()
894 {
895 LValue destination = m_phis.get(m_node->phi());
896
897 switch (m_node->child1().useKind()) {
898 case DoubleRepUse:
899 m_out.set(lowDouble(m_node->child1()), destination);
900 break;
901 case Int32Use:
902 m_out.set(lowInt32(m_node->child1()), destination);
903 break;
904 case Int52RepUse:
905 m_out.set(lowInt52(m_node->child1()), destination);
906 break;
907 case BooleanUse:
908 m_out.set(lowBoolean(m_node->child1()), destination);
909 break;
910 case CellUse:
911 m_out.set(lowCell(m_node->child1()), destination);
912 break;
913 case UntypedUse:
914 m_out.set(lowJSValue(m_node->child1()), destination);
915 break;
916 default:
917 DFG_CRASH(m_graph, m_node, "Bad use kind");
918 break;
919 }
920 }
921
922 void compilePhi()
923 {
924 LValue source = m_phis.get(m_node);
925
926 switch (m_node->flags() & NodeResultMask) {
927 case NodeResultDouble:
928 setDouble(m_out.get(source));
929 break;
930 case NodeResultInt32:
931 setInt32(m_out.get(source));
932 break;
933 case NodeResultInt52:
934 setInt52(m_out.get(source));
935 break;
936 case NodeResultBoolean:
937 setBoolean(m_out.get(source));
938 break;
939 case NodeResultJS:
940 setJSValue(m_out.get(source));
941 break;
942 default:
943 DFG_CRASH(m_graph, m_node, "Bad use kind");
944 break;
945 }
946 }
947
948 void compileDoubleConstant()
949 {
950 setDouble(m_out.constDouble(m_node->asNumber()));
951 }
952
953 void compileInt52Constant()
954 {
955 int64_t value = m_node->asMachineInt();
956
957 setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount));
958 setStrictInt52(m_out.constInt64(value));
959 }
960
961 void compileDoubleRep()
962 {
963 switch (m_node->child1().useKind()) {
964 case RealNumberUse: {
965 LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
966
967 LValue doubleValue = unboxDouble(value);
968
969 LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("DoubleRep RealNumberUse int case"));
970 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("DoubleRep continuation"));
971
972 ValueFromBlock fastResult = m_out.anchor(doubleValue);
973 m_out.branch(
974 m_out.doubleEqual(doubleValue, doubleValue),
975 usually(continuation), rarely(intCase));
976
977 LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
978
979 FTL_TYPE_CHECK(
980 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
981 isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble));
982 ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value)));
983 m_out.jump(continuation);
984
985 m_out.appendTo(continuation, lastNext);
986
987 setDouble(m_out.phi(m_out.doubleType, fastResult, slowResult));
988 return;
989 }
990
991 case NotCellUse:
992 case NumberUse: {
993 bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse;
994
995 LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation);
996
997 LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing int case"));
998 LBasicBlock doubleTesting = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing double case"));
999 LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing double case"));
1000 LBasicBlock nonDoubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing undefined case"));
1001 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToDouble unboxing continuation"));
1002
1003 m_out.branch(
1004 isNotInt32(value, provenType(m_node->child1())),
1005 unsure(doubleTesting), unsure(intCase));
1006
1007 LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting);
1008
1009 ValueFromBlock intToDouble = m_out.anchor(
1010 m_out.intToDouble(unboxInt32(value)));
1011 m_out.jump(continuation);
1012
1013 m_out.appendTo(doubleTesting, doubleCase);
1014 LValue valueIsNumber = isNumber(value, provenType(m_node->child1()));
1015 m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase));
1016
1017 m_out.appendTo(doubleCase, nonDoubleCase);
1018 ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value));
1019 m_out.jump(continuation);
1020
1021 if (shouldConvertNonNumber) {
1022 LBasicBlock undefinedCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble converting undefined case"));
1023 LBasicBlock testNullCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing null case"));
1024 LBasicBlock nullCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble converting null case"));
1025 LBasicBlock testBooleanTrueCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble testing boolean true case"));
1026 LBasicBlock convertBooleanTrueCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble convert boolean true case"));
1027 LBasicBlock convertBooleanFalseCase = FTL_NEW_BLOCK(m_out, ("jsValueToDouble convert boolean false case"));
1028
1029 m_out.appendTo(nonDoubleCase, undefinedCase);
1030 LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(ValueUndefined));
1031 m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase));
1032
1033 m_out.appendTo(undefinedCase, testNullCase);
1034 ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN));
1035 m_out.jump(continuation);
1036
1037 m_out.appendTo(testNullCase, nullCase);
1038 LValue valueIsNull = m_out.equal(value, m_out.constInt64(ValueNull));
1039 m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase));
1040
1041 m_out.appendTo(nullCase, testBooleanTrueCase);
1042 ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0));
1043 m_out.jump(continuation);
1044
1045 m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase);
1046 LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(ValueTrue));
1047 m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase));
1048
1049 m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase);
1050 ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1));
1051 m_out.jump(continuation);
1052
1053 m_out.appendTo(convertBooleanFalseCase, continuation);
1054
1055 LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(ValueFalse));
1056 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCell, valueIsNotBooleanFalse);
1057 ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0));
1058 m_out.jump(continuation);
1059
1060 m_out.appendTo(continuation, lastNext);
1061 setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse));
1062 return;
1063 }
1064 m_out.appendTo(nonDoubleCase, continuation);
1065 FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue);
1066 m_out.unreachable();
1067
1068 m_out.appendTo(continuation, lastNext);
1069
1070 setDouble(m_out.phi(m_out.doubleType, intToDouble, unboxedDouble));
1071 return;
1072 }
1073
1074 case Int52RepUse: {
1075 setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1())));
1076 return;
1077 }
1078
1079 default:
1080 DFG_CRASH(m_graph, m_node, "Bad use kind");
1081 }
1082 }
1083
1084 void compileDoubleAsInt32()
1085 {
1086 LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode()));
1087 setInt32(integerValue);
1088 }
1089
1090 void compileValueRep()
1091 {
1092 switch (m_node->child1().useKind()) {
1093 case DoubleRepUse: {
1094 LValue value = lowDouble(m_node->child1());
1095
1096 if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) {
1097 value = m_out.select(
1098 m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN));
1099 }
1100
1101 setJSValue(boxDouble(value));
1102 return;
1103 }
1104
1105 case Int52RepUse: {
1106 setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1())));
1107 return;
1108 }
1109
1110 default:
1111 DFG_CRASH(m_graph, m_node, "Bad use kind");
1112 }
1113 }
1114
1115 void compileInt52Rep()
1116 {
1117 switch (m_node->child1().useKind()) {
1118 case Int32Use:
1119 setStrictInt52(m_out.signExt(lowInt32(m_node->child1()), m_out.int64));
1120 return;
1121
1122 case MachineIntUse:
1123 setStrictInt52(
1124 jsValueToStrictInt52(
1125 m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1126 return;
1127
1128 case DoubleRepMachineIntUse:
1129 setStrictInt52(
1130 doubleToStrictInt52(
1131 m_node->child1(), lowDouble(m_node->child1())));
1132 return;
1133
1134 default:
1135 RELEASE_ASSERT_NOT_REACHED();
1136 }
1137 }
1138
1139 void compileValueToInt32()
1140 {
1141 switch (m_node->child1().useKind()) {
1142 case Int52RepUse:
1143 setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1())));
1144 break;
1145
1146 case DoubleRepUse:
1147 setInt32(doubleToInt32(lowDouble(m_node->child1())));
1148 break;
1149
1150 case NumberUse:
1151 case NotCellUse: {
1152 LoweredNodeValue value = m_int32Values.get(m_node->child1().node());
1153 if (isValid(value)) {
1154 setInt32(value.value());
1155 break;
1156 }
1157
1158 value = m_jsValueValues.get(m_node->child1().node());
1159 if (isValid(value)) {
1160 setInt32(numberOrNotCellToInt32(m_node->child1(), value.value()));
1161 break;
1162 }
1163
1164 // We'll basically just get here for constants. But it's good to have this
1165 // catch-all since we often add new representations into the mix.
1166 setInt32(
1167 numberOrNotCellToInt32(
1168 m_node->child1(),
1169 lowJSValue(m_node->child1(), ManualOperandSpeculation)));
1170 break;
1171 }
1172
1173 default:
1174 DFG_CRASH(m_graph, m_node, "Bad use kind");
1175 break;
1176 }
1177 }
1178
1179 void compileBooleanToNumber()
1180 {
1181 switch (m_node->child1().useKind()) {
1182 case BooleanUse: {
1183 setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), m_out.int32));
1184 return;
1185 }
1186
1187 case UntypedUse: {
1188 LValue value = lowJSValue(m_node->child1());
1189
1190 if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) {
1191 setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One));
1192 return;
1193 }
1194
1195 LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("BooleanToNumber boolean case"));
1196 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("BooleanToNumber continuation"));
1197
1198 ValueFromBlock notBooleanResult = m_out.anchor(value);
1199 m_out.branch(
1200 isBoolean(value, provenType(m_node->child1())),
1201 unsure(booleanCase), unsure(continuation));
1202
1203 LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation);
1204 ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr(
1205 m_out.zeroExt(unboxBoolean(value), m_out.int64), m_tagTypeNumber));
1206 m_out.jump(continuation);
1207
1208 m_out.appendTo(continuation, lastNext);
1209 setJSValue(m_out.phi(m_out.int64, booleanResult, notBooleanResult));
1210 return;
1211 }
1212
1213 default:
1214 RELEASE_ASSERT_NOT_REACHED();
1215 return;
1216 }
1217 }
1218
1219 void compileExtractOSREntryLocal()
1220 {
1221 EncodedJSValue* buffer = static_cast<EncodedJSValue*>(
1222 m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer());
1223 setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal())));
1224 }
1225
1226 void compileGetStack()
1227 {
1228 // GetLocals arise only for captured variables and arguments. For arguments, we might have
1229 // already loaded it.
1230 if (LValue value = m_loadedArgumentValues.get(m_node)) {
1231 setJSValue(value);
1232 return;
1233 }
1234
1235 StackAccessData* data = m_node->stackAccessData();
1236 AbstractValue& value = m_state.variables().operand(data->local);
1237
1238 DFG_ASSERT(m_graph, m_node, isConcrete(data->format));
1239 DFG_ASSERT(m_graph, m_node, data->format != FlushedDouble); // This just happens to not arise for GetStacks, right now. It would be trivial to support.
1240
1241 if (isInt32Speculation(value.m_type))
1242 setInt32(m_out.load32(payloadFor(data->machineLocal)));
1243 else
1244 setJSValue(m_out.load64(addressFor(data->machineLocal)));
1245 }
1246
1247 void compilePutStack()
1248 {
1249 StackAccessData* data = m_node->stackAccessData();
1250 switch (data->format) {
1251 case FlushedJSValue: {
1252 LValue value = lowJSValue(m_node->child1());
1253 m_out.store64(value, addressFor(data->machineLocal));
1254 break;
1255 }
1256
1257 case FlushedDouble: {
1258 LValue value = lowDouble(m_node->child1());
1259 m_out.storeDouble(value, addressFor(data->machineLocal));
1260 break;
1261 }
1262
1263 case FlushedInt32: {
1264 LValue value = lowInt32(m_node->child1());
1265 m_out.store32(value, payloadFor(data->machineLocal));
1266 break;
1267 }
1268
1269 case FlushedInt52: {
1270 LValue value = lowInt52(m_node->child1());
1271 m_out.store64(value, addressFor(data->machineLocal));
1272 break;
1273 }
1274
1275 case FlushedCell: {
1276 LValue value = lowCell(m_node->child1());
1277 m_out.store64(value, addressFor(data->machineLocal));
1278 break;
1279 }
1280
1281 case FlushedBoolean: {
1282 speculateBoolean(m_node->child1());
1283 m_out.store64(
1284 lowJSValue(m_node->child1(), ManualOperandSpeculation),
1285 addressFor(data->machineLocal));
1286 break;
1287 }
1288
1289 default:
1290 DFG_CRASH(m_graph, m_node, "Bad flush format");
1291 break;
1292 }
1293 }
1294
1295 void compileNoOp()
1296 {
1297 DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate);
1298 }
1299
1300 void compileToThis()
1301 {
1302 LValue value = lowJSValue(m_node->child1());
1303
1304 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToThis is cell case"));
1305 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ToThis slow case"));
1306 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToThis continuation"));
1307
1308 m_out.branch(
1309 isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase));
1310
1311 LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase);
1312 ValueFromBlock fastResult = m_out.anchor(value);
1313 m_out.branch(isType(value, FinalObjectType), usually(continuation), rarely(slowCase));
1314
1315 m_out.appendTo(slowCase, continuation);
1316 J_JITOperation_EJ function;
1317 if (m_graph.isStrictModeFor(m_node->origin.semantic))
1318 function = operationToThisStrict;
1319 else
1320 function = operationToThis;
1321 ValueFromBlock slowResult = m_out.anchor(
1322 vmCall(m_out.operation(function), m_callFrame, value));
1323 m_out.jump(continuation);
1324
1325 m_out.appendTo(continuation, lastNext);
1326 setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
1327 }
1328
1329 void compileValueAdd()
1330 {
1331 J_JITOperation_EJJ operation;
1332 if (!(provenType(m_node->child1()) & SpecFullNumber)
1333 && !(provenType(m_node->child2()) & SpecFullNumber))
1334 operation = operationValueAddNotNumber;
1335 else
1336 operation = operationValueAdd;
1337 setJSValue(vmCall(
1338 m_out.operation(operation), m_callFrame,
1339 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
1340 }
1341
1342 void compileArithAddOrSub()
1343 {
1344 bool isSub = m_node->op() == ArithSub;
1345 switch (m_node->binaryUseKind()) {
1346 case Int32Use: {
1347 LValue left = lowInt32(m_node->child1());
1348 LValue right = lowInt32(m_node->child2());
1349
1350 if (!shouldCheckOverflow(m_node->arithMode())) {
1351 setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right));
1352 break;
1353 }
1354
1355 LValue result;
1356 if (!isSub) {
1357 result = m_out.addWithOverflow32(left, right);
1358
1359 if (doesKill(m_node->child2())) {
1360 addAvailableRecovery(
1361 m_node->child2(), SubRecovery,
1362 m_out.extractValue(result, 0), left, ValueFormatInt32);
1363 } else if (doesKill(m_node->child1())) {
1364 addAvailableRecovery(
1365 m_node->child1(), SubRecovery,
1366 m_out.extractValue(result, 0), right, ValueFormatInt32);
1367 }
1368 } else {
1369 result = m_out.subWithOverflow32(left, right);
1370
1371 if (doesKill(m_node->child2())) {
1372 // result = left - right
1373 // result - left = -right
1374 // right = left - result
1375 addAvailableRecovery(
1376 m_node->child2(), SubRecovery,
1377 left, m_out.extractValue(result, 0), ValueFormatInt32);
1378 } else if (doesKill(m_node->child1())) {
1379 // result = left - right
1380 // result + right = left
1381 addAvailableRecovery(
1382 m_node->child1(), AddRecovery,
1383 m_out.extractValue(result, 0), right, ValueFormatInt32);
1384 }
1385 }
1386
1387 speculate(Overflow, noValue(), 0, m_out.extractValue(result, 1));
1388 setInt32(m_out.extractValue(result, 0));
1389 break;
1390 }
1391
1392 case Int52RepUse: {
1393 if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)
1394 && !abstractValue(m_node->child2()).couldBeType(SpecInt52)) {
1395 Int52Kind kind;
1396 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1397 LValue right = lowInt52(m_node->child2(), kind);
1398 setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind);
1399 break;
1400 }
1401
1402 LValue left = lowInt52(m_node->child1());
1403 LValue right = lowInt52(m_node->child2());
1404
1405 LValue result;
1406 if (!isSub) {
1407 result = m_out.addWithOverflow64(left, right);
1408
1409 if (doesKill(m_node->child2())) {
1410 addAvailableRecovery(
1411 m_node->child2(), SubRecovery,
1412 m_out.extractValue(result, 0), left, ValueFormatInt52);
1413 } else if (doesKill(m_node->child1())) {
1414 addAvailableRecovery(
1415 m_node->child1(), SubRecovery,
1416 m_out.extractValue(result, 0), right, ValueFormatInt52);
1417 }
1418 } else {
1419 result = m_out.subWithOverflow64(left, right);
1420
1421 if (doesKill(m_node->child2())) {
1422 // result = left - right
1423 // result - left = -right
1424 // right = left - result
1425 addAvailableRecovery(
1426 m_node->child2(), SubRecovery,
1427 left, m_out.extractValue(result, 0), ValueFormatInt52);
1428 } else if (doesKill(m_node->child1())) {
1429 // result = left - right
1430 // result + right = left
1431 addAvailableRecovery(
1432 m_node->child1(), AddRecovery,
1433 m_out.extractValue(result, 0), right, ValueFormatInt52);
1434 }
1435 }
1436
1437 speculate(Int52Overflow, noValue(), 0, m_out.extractValue(result, 1));
1438 setInt52(m_out.extractValue(result, 0));
1439 break;
1440 }
1441
1442 case DoubleRepUse: {
1443 LValue C1 = lowDouble(m_node->child1());
1444 LValue C2 = lowDouble(m_node->child2());
1445
1446 setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2));
1447 break;
1448 }
1449
1450 default:
1451 DFG_CRASH(m_graph, m_node, "Bad use kind");
1452 break;
1453 }
1454 }
1455
1456 void compileArithClz32()
1457 {
1458 LValue operand = lowInt32(m_node->child1());
1459 LValue isZeroUndef = m_out.booleanFalse;
1460 setInt32(m_out.ctlz32(operand, isZeroUndef));
1461 }
1462
1463 void compileArithMul()
1464 {
1465 switch (m_node->binaryUseKind()) {
1466 case Int32Use: {
1467 LValue left = lowInt32(m_node->child1());
1468 LValue right = lowInt32(m_node->child2());
1469
1470 LValue result;
1471
1472 if (!shouldCheckOverflow(m_node->arithMode()))
1473 result = m_out.mul(left, right);
1474 else {
1475 LValue overflowResult = m_out.mulWithOverflow32(left, right);
1476 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1477 result = m_out.extractValue(overflowResult, 0);
1478 }
1479
1480 if (shouldCheckNegativeZero(m_node->arithMode())) {
1481 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1482 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1483
1484 m_out.branch(
1485 m_out.notZero32(result), usually(continuation), rarely(slowCase));
1486
1487 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1488 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int32Zero), m_out.lessThan(right, m_out.int32Zero));
1489 speculate(NegativeZero, noValue(), 0, cond);
1490 m_out.jump(continuation);
1491 m_out.appendTo(continuation, lastNext);
1492 }
1493
1494 setInt32(result);
1495 break;
1496 }
1497
1498 case Int52RepUse: {
1499 Int52Kind kind;
1500 LValue left = lowWhicheverInt52(m_node->child1(), kind);
1501 LValue right = lowInt52(m_node->child2(), opposite(kind));
1502
1503 LValue overflowResult = m_out.mulWithOverflow64(left, right);
1504 speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1505 LValue result = m_out.extractValue(overflowResult, 0);
1506
1507 if (shouldCheckNegativeZero(m_node->arithMode())) {
1508 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArithMul slow case"));
1509 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMul continuation"));
1510
1511 m_out.branch(
1512 m_out.notZero64(result), usually(continuation), rarely(slowCase));
1513
1514 LBasicBlock lastNext = m_out.appendTo(slowCase, continuation);
1515 LValue cond = m_out.bitOr(m_out.lessThan(left, m_out.int64Zero), m_out.lessThan(right, m_out.int64Zero));
1516 speculate(NegativeZero, noValue(), 0, cond);
1517 m_out.jump(continuation);
1518 m_out.appendTo(continuation, lastNext);
1519 }
1520
1521 setInt52(result);
1522 break;
1523 }
1524
1525 case DoubleRepUse: {
1526 setDouble(
1527 m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1528 break;
1529 }
1530
1531 default:
1532 DFG_CRASH(m_graph, m_node, "Bad use kind");
1533 break;
1534 }
1535 }
1536
1537 void compileArithDiv()
1538 {
1539 switch (m_node->binaryUseKind()) {
1540 case Int32Use: {
1541 LValue numerator = lowInt32(m_node->child1());
1542 LValue denominator = lowInt32(m_node->child2());
1543
1544 LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithDiv unsafe denominator"));
1545 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithDiv continuation"));
1546 LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithDiv done"));
1547
1548 Vector<ValueFromBlock, 3> results;
1549
1550 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1551
1552 m_out.branch(
1553 m_out.above(adjustedDenominator, m_out.int32One),
1554 usually(continuation), rarely(unsafeDenominator));
1555
1556 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1557
1558 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1559
1560 if (shouldCheckOverflow(m_node->arithMode())) {
1561 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1562 speculate(Overflow, noValue(), 0, cond);
1563 m_out.jump(continuation);
1564 } else {
1565 // This is the case where we convert the result to an int after we're done. So,
1566 // if the denominator is zero, then the result should be zero.
1567 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1568 // check above) and the numerator is -2^31 then the result should be -2^31.
1569
1570 LBasicBlock divByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv divide by zero"));
1571 LBasicBlock notDivByZero = FTL_NEW_BLOCK(m_out, ("ArithDiv not divide by zero"));
1572 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithDiv -2^31/-1"));
1573
1574 m_out.branch(
1575 m_out.isZero32(denominator), rarely(divByZero), usually(notDivByZero));
1576
1577 m_out.appendTo(divByZero, notDivByZero);
1578 results.append(m_out.anchor(m_out.int32Zero));
1579 m_out.jump(done);
1580
1581 m_out.appendTo(notDivByZero, neg2ToThe31ByNeg1);
1582 m_out.branch(
1583 m_out.equal(numerator, neg2ToThe31),
1584 rarely(neg2ToThe31ByNeg1), usually(continuation));
1585
1586 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1587 results.append(m_out.anchor(neg2ToThe31));
1588 m_out.jump(done);
1589 }
1590
1591 m_out.appendTo(continuation, done);
1592
1593 if (shouldCheckNegativeZero(m_node->arithMode())) {
1594 LBasicBlock zeroNumerator = FTL_NEW_BLOCK(m_out, ("ArithDiv zero numerator"));
1595 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithDiv numerator continuation"));
1596
1597 m_out.branch(
1598 m_out.isZero32(numerator),
1599 rarely(zeroNumerator), usually(numeratorContinuation));
1600
1601 LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation);
1602
1603 speculate(
1604 NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero));
1605
1606 m_out.jump(numeratorContinuation);
1607
1608 m_out.appendTo(numeratorContinuation, innerLastNext);
1609 }
1610
1611 LValue result = m_out.div(numerator, denominator);
1612
1613 if (shouldCheckOverflow(m_node->arithMode())) {
1614 speculate(
1615 Overflow, noValue(), 0,
1616 m_out.notEqual(m_out.mul(result, denominator), numerator));
1617 }
1618
1619 results.append(m_out.anchor(result));
1620 m_out.jump(done);
1621
1622 m_out.appendTo(done, lastNext);
1623
1624 setInt32(m_out.phi(m_out.int32, results));
1625 break;
1626 }
1627
1628 case DoubleRepUse: {
1629 setDouble(m_out.doubleDiv(
1630 lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1631 break;
1632 }
1633
1634 default:
1635 DFG_CRASH(m_graph, m_node, "Bad use kind");
1636 break;
1637 }
1638 }
1639
1640 void compileArithMod()
1641 {
1642 switch (m_node->binaryUseKind()) {
1643 case Int32Use: {
1644 LValue numerator = lowInt32(m_node->child1());
1645 LValue denominator = lowInt32(m_node->child2());
1646
1647 LBasicBlock unsafeDenominator = FTL_NEW_BLOCK(m_out, ("ArithMod unsafe denominator"));
1648 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMod continuation"));
1649 LBasicBlock done = FTL_NEW_BLOCK(m_out, ("ArithMod done"));
1650
1651 Vector<ValueFromBlock, 3> results;
1652
1653 LValue adjustedDenominator = m_out.add(denominator, m_out.int32One);
1654
1655 m_out.branch(
1656 m_out.above(adjustedDenominator, m_out.int32One),
1657 usually(continuation), rarely(unsafeDenominator));
1658
1659 LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation);
1660
1661 LValue neg2ToThe31 = m_out.constInt32(-2147483647-1);
1662
1663 // FIXME: -2^31 / -1 will actually yield negative zero, so we could have a
1664 // separate case for that. But it probably doesn't matter so much.
1665 if (shouldCheckOverflow(m_node->arithMode())) {
1666 LValue cond = m_out.bitOr(m_out.isZero32(denominator), m_out.equal(numerator, neg2ToThe31));
1667 speculate(Overflow, noValue(), 0, cond);
1668 m_out.jump(continuation);
1669 } else {
1670 // This is the case where we convert the result to an int after we're done. So,
1671 // if the denominator is zero, then the result should be result should be zero.
1672 // If the denominator is not zero (i.e. it's -1 because we're guarded by the
1673 // check above) and the numerator is -2^31 then the result should be -2^31.
1674
1675 LBasicBlock modByZero = FTL_NEW_BLOCK(m_out, ("ArithMod modulo by zero"));
1676 LBasicBlock notModByZero = FTL_NEW_BLOCK(m_out, ("ArithMod not modulo by zero"));
1677 LBasicBlock neg2ToThe31ByNeg1 = FTL_NEW_BLOCK(m_out, ("ArithMod -2^31/-1"));
1678
1679 m_out.branch(
1680 m_out.isZero32(denominator), rarely(modByZero), usually(notModByZero));
1681
1682 m_out.appendTo(modByZero, notModByZero);
1683 results.append(m_out.anchor(m_out.int32Zero));
1684 m_out.jump(done);
1685
1686 m_out.appendTo(notModByZero, neg2ToThe31ByNeg1);
1687 m_out.branch(
1688 m_out.equal(numerator, neg2ToThe31),
1689 rarely(neg2ToThe31ByNeg1), usually(continuation));
1690
1691 m_out.appendTo(neg2ToThe31ByNeg1, continuation);
1692 results.append(m_out.anchor(m_out.int32Zero));
1693 m_out.jump(done);
1694 }
1695
1696 m_out.appendTo(continuation, done);
1697
1698 LValue remainder = m_out.rem(numerator, denominator);
1699
1700 if (shouldCheckNegativeZero(m_node->arithMode())) {
1701 LBasicBlock negativeNumerator = FTL_NEW_BLOCK(m_out, ("ArithMod negative numerator"));
1702 LBasicBlock numeratorContinuation = FTL_NEW_BLOCK(m_out, ("ArithMod numerator continuation"));
1703
1704 m_out.branch(
1705 m_out.lessThan(numerator, m_out.int32Zero),
1706 unsure(negativeNumerator), unsure(numeratorContinuation));
1707
1708 LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation);
1709
1710 speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder));
1711
1712 m_out.jump(numeratorContinuation);
1713
1714 m_out.appendTo(numeratorContinuation, innerLastNext);
1715 }
1716
1717 results.append(m_out.anchor(remainder));
1718 m_out.jump(done);
1719
1720 m_out.appendTo(done, lastNext);
1721
1722 setInt32(m_out.phi(m_out.int32, results));
1723 break;
1724 }
1725
1726 case DoubleRepUse: {
1727 setDouble(
1728 m_out.doubleRem(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
1729 break;
1730 }
1731
1732 default:
1733 DFG_CRASH(m_graph, m_node, "Bad use kind");
1734 break;
1735 }
1736 }
1737
1738 void compileArithMinOrMax()
1739 {
1740 switch (m_node->binaryUseKind()) {
1741 case Int32Use: {
1742 LValue left = lowInt32(m_node->child1());
1743 LValue right = lowInt32(m_node->child2());
1744
1745 setInt32(
1746 m_out.select(
1747 m_node->op() == ArithMin
1748 ? m_out.lessThan(left, right)
1749 : m_out.lessThan(right, left),
1750 left, right));
1751 break;
1752 }
1753
1754 case DoubleRepUse: {
1755 LValue left = lowDouble(m_node->child1());
1756 LValue right = lowDouble(m_node->child2());
1757
1758 LBasicBlock notLessThan = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax not less than"));
1759 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithMin/ArithMax continuation"));
1760
1761 Vector<ValueFromBlock, 2> results;
1762
1763 results.append(m_out.anchor(left));
1764 m_out.branch(
1765 m_node->op() == ArithMin
1766 ? m_out.doubleLessThan(left, right)
1767 : m_out.doubleGreaterThan(left, right),
1768 unsure(continuation), unsure(notLessThan));
1769
1770 LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation);
1771 results.append(m_out.anchor(m_out.select(
1772 m_node->op() == ArithMin
1773 ? m_out.doubleGreaterThanOrEqual(left, right)
1774 : m_out.doubleLessThanOrEqual(left, right),
1775 right, m_out.constDouble(PNaN))));
1776 m_out.jump(continuation);
1777
1778 m_out.appendTo(continuation, lastNext);
1779 setDouble(m_out.phi(m_out.doubleType, results));
1780 break;
1781 }
1782
1783 default:
1784 DFG_CRASH(m_graph, m_node, "Bad use kind");
1785 break;
1786 }
1787 }
1788
1789 void compileArithAbs()
1790 {
1791 switch (m_node->child1().useKind()) {
1792 case Int32Use: {
1793 LValue value = lowInt32(m_node->child1());
1794
1795 LValue mask = m_out.aShr(value, m_out.constInt32(31));
1796 LValue result = m_out.bitXor(mask, m_out.add(mask, value));
1797
1798 speculate(Overflow, noValue(), 0, m_out.equal(result, m_out.constInt32(1 << 31)));
1799
1800 setInt32(result);
1801 break;
1802 }
1803
1804 case DoubleRepUse: {
1805 setDouble(m_out.doubleAbs(lowDouble(m_node->child1())));
1806 break;
1807 }
1808
1809 default:
1810 DFG_CRASH(m_graph, m_node, "Bad use kind");
1811 break;
1812 }
1813 }
1814
1815 void compileArithSin() { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); }
1816
1817 void compileArithCos() { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); }
1818
1819 void compileArithPow()
1820 {
1821 // FIXME: investigate llvm.powi to better understand its performance characteristics.
1822 // It might be better to have the inline loop in DFG too.
1823 if (m_node->child2().useKind() == Int32Use)
1824 setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2())));
1825 else {
1826 LValue base = lowDouble(m_node->child1());
1827 LValue exponent = lowDouble(m_node->child2());
1828
1829 LBasicBlock integerExponentIsSmallBlock = FTL_NEW_BLOCK(m_out, ("ArithPow test integer exponent is small."));
1830 LBasicBlock integerExponentPowBlock = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, (int)double)."));
1831 LBasicBlock doubleExponentPowBlockEntry = FTL_NEW_BLOCK(m_out, ("ArithPow pow(double, double)."));
1832 LBasicBlock nanExceptionExponentIsInfinity = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check exponent is infinity."));
1833 LBasicBlock nanExceptionBaseIsOne = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, check base is one."));
1834 LBasicBlock powBlock = FTL_NEW_BLOCK(m_out, ("ArithPow regular pow"));
1835 LBasicBlock nanExceptionResultIsNaN = FTL_NEW_BLOCK(m_out, ("ArithPow NaN Exception, result is NaN."));
1836 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithPow continuation"));
1837
1838 LValue integerExponent = m_out.fpToInt32(exponent);
1839 LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent);
1840 LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble);
1841 m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry));
1842
1843 LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock);
1844 LValue integerExponentBelow1000 = m_out.below(integerExponent, m_out.constInt32(1000));
1845 m_out.branch(integerExponentBelow1000, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry));
1846
1847 m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry);
1848 ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent));
1849 m_out.jump(continuation);
1850
1851 // If y is NaN, the result is NaN.
1852 m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionExponentIsInfinity);
1853 LValue exponentIsNaN;
1854 if (provenType(m_node->child2()) & SpecDoubleNaN)
1855 exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent);
1856 else
1857 exponentIsNaN = m_out.booleanFalse;
1858 m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionExponentIsInfinity));
1859
1860 // If abs(x) is 1 and y is +infinity, the result is NaN.
1861 // If abs(x) is 1 and y is -infinity, the result is NaN.
1862 m_out.appendTo(nanExceptionExponentIsInfinity, nanExceptionBaseIsOne);
1863 LValue absoluteExponent = m_out.doubleAbs(exponent);
1864 LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits<double>::infinity()));
1865 m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionBaseIsOne), usually(powBlock));
1866
1867 m_out.appendTo(nanExceptionBaseIsOne, powBlock);
1868 LValue absoluteBase = m_out.doubleAbs(base);
1869 LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1));
1870 m_out.branch(absoluteBaseIsOne, unsure(nanExceptionResultIsNaN), unsure(powBlock));
1871
1872 m_out.appendTo(powBlock, nanExceptionResultIsNaN);
1873 ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent));
1874 m_out.jump(continuation);
1875
1876 m_out.appendTo(nanExceptionResultIsNaN, continuation);
1877 ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN));
1878 m_out.jump(continuation);
1879
1880 m_out.appendTo(continuation, lastNext);
1881 setDouble(m_out.phi(m_out.doubleType, powDoubleIntResult, powResult, pureNan));
1882 }
1883 }
1884
1885 void compileArithRound()
1886 {
1887 LBasicBlock realPartIsMoreThanHalf = FTL_NEW_BLOCK(m_out, ("ArithRound should round down"));
1888 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArithRound continuation"));
1889
1890 LValue value = lowDouble(m_node->child1());
1891 LValue integerValue = m_out.ceil64(value);
1892 ValueFromBlock integerValueResult = m_out.anchor(integerValue);
1893
1894 LValue realPart = m_out.doubleSub(integerValue, value);
1895
1896 m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation));
1897
1898 LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation);
1899 LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1));
1900 ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown);
1901 m_out.jump(continuation);
1902 m_out.appendTo(continuation, lastNext);
1903
1904 LValue result = m_out.phi(m_out.doubleType, integerValueResult, integerValueRoundedDownResult);
1905
1906 if (producesInteger(m_node->arithRoundingMode())) {
1907 LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()));
1908 setInt32(integerValue);
1909 } else
1910 setDouble(result);
1911 }
1912
1913 void compileArithSqrt() { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); }
1914
1915 void compileArithLog() { setDouble(m_out.doubleLog(lowDouble(m_node->child1()))); }
1916
1917 void compileArithFRound()
1918 {
1919 LValue floatValue = m_out.fpCast(lowDouble(m_node->child1()), m_out.floatType);
1920 setDouble(m_out.fpCast(floatValue, m_out.doubleType));
1921 }
1922
1923 void compileArithNegate()
1924 {
1925 switch (m_node->child1().useKind()) {
1926 case Int32Use: {
1927 LValue value = lowInt32(m_node->child1());
1928
1929 LValue result;
1930 if (!shouldCheckOverflow(m_node->arithMode()))
1931 result = m_out.neg(value);
1932 else if (!shouldCheckNegativeZero(m_node->arithMode())) {
1933 // We don't have a negate-with-overflow intrinsic. Hopefully this
1934 // does the trick, though.
1935 LValue overflowResult = m_out.subWithOverflow32(m_out.int32Zero, value);
1936 speculate(Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1937 result = m_out.extractValue(overflowResult, 0);
1938 } else {
1939 speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff)));
1940 result = m_out.neg(value);
1941 }
1942
1943 setInt32(result);
1944 break;
1945 }
1946
1947 case Int52RepUse: {
1948 if (!abstractValue(m_node->child1()).couldBeType(SpecInt52)) {
1949 Int52Kind kind;
1950 LValue value = lowWhicheverInt52(m_node->child1(), kind);
1951 LValue result = m_out.neg(value);
1952 if (shouldCheckNegativeZero(m_node->arithMode()))
1953 speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1954 setInt52(result, kind);
1955 break;
1956 }
1957
1958 LValue value = lowInt52(m_node->child1());
1959 LValue overflowResult = m_out.subWithOverflow64(m_out.int64Zero, value);
1960 speculate(Int52Overflow, noValue(), 0, m_out.extractValue(overflowResult, 1));
1961 LValue result = m_out.extractValue(overflowResult, 0);
1962 speculate(NegativeZero, noValue(), 0, m_out.isZero64(result));
1963 setInt52(result);
1964 break;
1965 }
1966
1967 case DoubleRepUse: {
1968 setDouble(m_out.doubleNeg(lowDouble(m_node->child1())));
1969 break;
1970 }
1971
1972 default:
1973 DFG_CRASH(m_graph, m_node, "Bad use kind");
1974 break;
1975 }
1976 }
1977
1978 void compileBitAnd()
1979 {
1980 setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1981 }
1982
1983 void compileBitOr()
1984 {
1985 setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1986 }
1987
1988 void compileBitXor()
1989 {
1990 setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
1991 }
1992
1993 void compileBitRShift()
1994 {
1995 setInt32(m_out.aShr(
1996 lowInt32(m_node->child1()),
1997 m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
1998 }
1999
2000 void compileBitLShift()
2001 {
2002 setInt32(m_out.shl(
2003 lowInt32(m_node->child1()),
2004 m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2005 }
2006
2007 void compileBitURShift()
2008 {
2009 setInt32(m_out.lShr(
2010 lowInt32(m_node->child1()),
2011 m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31))));
2012 }
2013
2014 void compileUInt32ToNumber()
2015 {
2016 LValue value = lowInt32(m_node->child1());
2017
2018 if (doesOverflow(m_node->arithMode())) {
2019 setDouble(m_out.unsignedToDouble(value));
2020 return;
2021 }
2022
2023 speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero));
2024 setInt32(value);
2025 }
2026
2027 void compileCheckStructure()
2028 {
2029 LValue cell = lowCell(m_node->child1());
2030
2031 ExitKind exitKind;
2032 if (m_node->child1()->hasConstant())
2033 exitKind = BadConstantCache;
2034 else
2035 exitKind = BadCache;
2036
2037 LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2038
2039 checkStructure(
2040 structureID, jsValueValue(cell), exitKind, m_node->structureSet(),
2041 [this] (Structure* structure) {
2042 return weakStructureID(structure);
2043 });
2044 }
2045
2046 void compileCheckCell()
2047 {
2048 LValue cell = lowCell(m_node->child1());
2049
2050 speculate(
2051 BadCell, jsValueValue(cell), m_node->child1().node(),
2052 m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell())));
2053 }
2054
2055 void compileCheckBadCell()
2056 {
2057 terminate(BadCell);
2058 }
2059
2060 void compileCheckNotEmpty()
2061 {
2062 speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1())));
2063 }
2064
2065 void compileGetExecutable()
2066 {
2067 LValue cell = lowCell(m_node->child1());
2068 speculateFunction(m_node->child1(), cell);
2069 setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable));
2070 }
2071
2072 void compileArrayifyToStructure()
2073 {
2074 LValue cell = lowCell(m_node->child1());
2075 LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0;
2076
2077 LBasicBlock unexpectedStructure = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure unexpected structure"));
2078 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayifyToStructure continuation"));
2079
2080 LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2081
2082 m_out.branch(
2083 m_out.notEqual(structureID, weakStructureID(m_node->structure())),
2084 rarely(unexpectedStructure), usually(continuation));
2085
2086 LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation);
2087
2088 if (property) {
2089 switch (m_node->arrayMode().type()) {
2090 case Array::Int32:
2091 case Array::Double:
2092 case Array::Contiguous:
2093 speculate(
2094 Uncountable, noValue(), 0,
2095 m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX)));
2096 break;
2097 default:
2098 break;
2099 }
2100 }
2101
2102 switch (m_node->arrayMode().type()) {
2103 case Array::Int32:
2104 vmCall(m_out.operation(operationEnsureInt32), m_callFrame, cell);
2105 break;
2106 case Array::Double:
2107 vmCall(m_out.operation(operationEnsureDouble), m_callFrame, cell);
2108 break;
2109 case Array::Contiguous:
2110 vmCall(m_out.operation(operationEnsureContiguous), m_callFrame, cell);
2111 break;
2112 case Array::ArrayStorage:
2113 case Array::SlowPutArrayStorage:
2114 vmCall(m_out.operation(operationEnsureArrayStorage), m_callFrame, cell);
2115 break;
2116 default:
2117 DFG_CRASH(m_graph, m_node, "Bad array type");
2118 break;
2119 }
2120
2121 structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
2122 speculate(
2123 BadIndexingType, jsValueValue(cell), 0,
2124 m_out.notEqual(structureID, weakStructureID(m_node->structure())));
2125 m_out.jump(continuation);
2126
2127 m_out.appendTo(continuation, lastNext);
2128 }
2129
2130 void compilePutStructure()
2131 {
2132 m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node);
2133
2134 Structure* oldStructure = m_node->transition()->previous;
2135 Structure* newStructure = m_node->transition()->next;
2136 ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType());
2137 ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags());
2138 ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type());
2139
2140 LValue cell = lowCell(m_node->child1());
2141 m_out.store32(
2142 weakStructureID(newStructure),
2143 cell, m_heaps.JSCell_structureID);
2144 }
2145
2146 void compileGetById()
2147 {
2148 // Pretty much the only reason why we don't also support GetByIdFlush is because:
2149 // https://bugs.webkit.org/show_bug.cgi?id=125711
2150
2151 switch (m_node->child1().useKind()) {
2152 case CellUse: {
2153 setJSValue(getById(lowCell(m_node->child1())));
2154 return;
2155 }
2156
2157 case UntypedUse: {
2158 // This is pretty weird, since we duplicate the slow path both here and in the
2159 // code generated by the IC. We should investigate making this less bad.
2160 // https://bugs.webkit.org/show_bug.cgi?id=127830
2161 LValue value = lowJSValue(m_node->child1());
2162
2163 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped cell case"));
2164 LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("GetById untyped not cell case"));
2165 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetById untyped continuation"));
2166
2167 m_out.branch(
2168 isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase));
2169
2170 LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase);
2171 ValueFromBlock cellResult = m_out.anchor(getById(value));
2172 m_out.jump(continuation);
2173
2174 m_out.appendTo(notCellCase, continuation);
2175 ValueFromBlock notCellResult = m_out.anchor(vmCall(
2176 m_out.operation(operationGetByIdGeneric),
2177 m_callFrame, value,
2178 m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])));
2179 m_out.jump(continuation);
2180
2181 m_out.appendTo(continuation, lastNext);
2182 setJSValue(m_out.phi(m_out.int64, cellResult, notCellResult));
2183 return;
2184 }
2185
2186 default:
2187 DFG_CRASH(m_graph, m_node, "Bad use kind");
2188 return;
2189 }
2190 }
2191
2192 void compilePutById()
2193 {
2194 // See above; CellUse is easier so we do only that for now.
2195 ASSERT(m_node->child1().useKind() == CellUse);
2196
2197 LValue base = lowCell(m_node->child1());
2198 LValue value = lowJSValue(m_node->child2());
2199 auto uid = m_graph.identifiers()[m_node->identifierNumber()];
2200
2201 // Arguments: id, bytes, target, numArgs, args...
2202 unsigned stackmapID = m_stackmapIDs++;
2203
2204 if (verboseCompilationEnabled())
2205 dataLog(" Emitting PutById patchpoint with stackmap #", stackmapID, "\n");
2206
2207 LValue call = m_out.call(
2208 m_out.patchpointVoidIntrinsic(),
2209 m_out.constInt64(stackmapID), m_out.constInt32(sizeOfPutById()),
2210 constNull(m_out.ref8), m_out.constInt32(2), base, value);
2211 setInstructionCallingConvention(call, LLVMAnyRegCallConv);
2212
2213 m_ftlState.putByIds.append(PutByIdDescriptor(
2214 stackmapID, m_node->origin.semantic, uid,
2215 m_graph.executableFor(m_node->origin.semantic)->ecmaMode(),
2216 m_node->op() == PutByIdDirect ? Direct : NotDirect));
2217 }
2218
2219 void compileGetButterfly()
2220 {
2221 setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly));
2222 }
2223
2224 void compileConstantStoragePointer()
2225 {
2226 setStorage(m_out.constIntPtr(m_node->storagePointer()));
2227 }
2228
2229 void compileGetIndexedPropertyStorage()
2230 {
2231 LValue cell = lowCell(m_node->child1());
2232
2233 if (m_node->arrayMode().type() == Array::String) {
2234 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String slow case"));
2235 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetIndexedPropertyStorage String continuation"));
2236
2237 ValueFromBlock fastResult = m_out.anchor(
2238 m_out.loadPtr(cell, m_heaps.JSString_value));
2239
2240 m_out.branch(
2241 m_out.notNull(fastResult.value()), usually(continuation), rarely(slowPath));
2242
2243 LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
2244
2245 ValueFromBlock slowResult = m_out.anchor(
2246 vmCall(m_out.operation(operationResolveRope), m_callFrame, cell));
2247
2248 m_out.jump(continuation);
2249
2250 m_out.appendTo(continuation, lastNext);
2251
2252 setStorage(m_out.loadPtr(m_out.phi(m_out.intPtr, fastResult, slowResult), m_heaps.StringImpl_data));
2253 return;
2254 }
2255
2256 setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector));
2257 }
2258
2259 void compileCheckArray()
2260 {
2261 Edge edge = m_node->child1();
2262 LValue cell = lowCell(edge);
2263
2264 if (m_node->arrayMode().alreadyChecked(m_graph, m_node, abstractValue(edge)))
2265 return;
2266
2267 speculate(
2268 BadIndexingType, jsValueValue(cell), 0,
2269 m_out.bitNot(isArrayType(cell, m_node->arrayMode())));
2270 }
2271
2272 void compileGetTypedArrayByteOffset()
2273 {
2274 LValue basePtr = lowCell(m_node->child1());
2275
2276 LBasicBlock simpleCase = FTL_NEW_BLOCK(m_out, ("wasteless typed array"));
2277 LBasicBlock wastefulCase = FTL_NEW_BLOCK(m_out, ("wasteful typed array"));
2278 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("continuation branch"));
2279
2280 LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode);
2281 m_out.branch(
2282 m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)),
2283 unsure(simpleCase), unsure(wastefulCase));
2284
2285 // begin simple case
2286 LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase);
2287
2288 ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0));
2289
2290 m_out.jump(continuation);
2291
2292 // begin wasteful case
2293 m_out.appendTo(wastefulCase, continuation);
2294
2295 LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector);
2296 LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly);
2297 LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer);
2298 LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data);
2299
2300 ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr));
2301
2302 m_out.jump(continuation);
2303 m_out.appendTo(continuation, lastNext);
2304
2305 // output
2306 setInt32(m_out.castToInt32(m_out.phi(m_out.intPtr, simpleOut, wastefulOut)));
2307 }
2308
2309 void compileGetArrayLength()
2310 {
2311 switch (m_node->arrayMode().type()) {
2312 case Array::Int32:
2313 case Array::Double:
2314 case Array::Contiguous: {
2315 setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength));
2316 return;
2317 }
2318
2319 case Array::String: {
2320 LValue string = lowCell(m_node->child1());
2321 setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length));
2322 return;
2323 }
2324
2325 case Array::DirectArguments: {
2326 LValue arguments = lowCell(m_node->child1());
2327 speculate(
2328 ExoticObjectMode, noValue(), nullptr,
2329 m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_overrides)));
2330 setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length));
2331 return;
2332 }
2333
2334 case Array::ScopedArguments: {
2335 LValue arguments = lowCell(m_node->child1());
2336 speculate(
2337 ExoticObjectMode, noValue(), nullptr,
2338 m_out.notZero8(m_out.load8(arguments, m_heaps.ScopedArguments_overrodeThings)));
2339 setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength));
2340 return;
2341 }
2342
2343 default:
2344 if (isTypedView(m_node->arrayMode().typedArrayType())) {
2345 setInt32(
2346 m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length));
2347 return;
2348 }
2349
2350 DFG_CRASH(m_graph, m_node, "Bad array type");
2351 return;
2352 }
2353 }
2354
2355 void compileCheckInBounds()
2356 {
2357 speculate(
2358 OutOfBounds, noValue(), 0,
2359 m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
2360 }
2361
2362 void compileGetByVal()
2363 {
2364 switch (m_node->arrayMode().type()) {
2365 case Array::Int32:
2366 case Array::Contiguous: {
2367 LValue index = lowInt32(m_node->child2());
2368 LValue storage = lowStorage(m_node->child3());
2369
2370 IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
2371 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
2372
2373 if (m_node->arrayMode().isInBounds()) {
2374 LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2()));
2375 LValue isHole = m_out.isZero64(result);
2376 if (m_node->arrayMode().isSaneChain()) {
2377 DFG_ASSERT(
2378 m_graph, m_node, m_node->arrayMode().type() == Array::Contiguous);
2379 result = m_out.select(
2380 isHole, m_out.constInt64(JSValue::encode(jsUndefined())), result);
2381 } else
2382 speculate(LoadFromHole, noValue(), 0, isHole);
2383 setJSValue(result);
2384 return;
2385 }
2386
2387 LValue base = lowCell(m_node->child1());
2388
2389 LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous fast case"));
2390 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous slow case"));
2391 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal int/contiguous continuation"));
2392
2393 m_out.branch(
2394 m_out.aboveOrEqual(
2395 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2396 rarely(slowCase), usually(fastCase));
2397
2398 LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
2399
2400 ValueFromBlock fastResult = m_out.anchor(
2401 m_out.load64(baseIndex(heap, storage, index, m_node->child2())));
2402 m_out.branch(
2403 m_out.isZero64(fastResult.value()), rarely(slowCase), usually(continuation));
2404
2405 m_out.appendTo(slowCase, continuation);
2406 ValueFromBlock slowResult = m_out.anchor(
2407 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2408 m_out.jump(continuation);
2409
2410 m_out.appendTo(continuation, lastNext);
2411 setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2412 return;
2413 }
2414
2415 case Array::Double: {
2416 LValue index = lowInt32(m_node->child2());
2417 LValue storage = lowStorage(m_node->child3());
2418
2419 IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
2420
2421 if (m_node->arrayMode().isInBounds()) {
2422 LValue result = m_out.loadDouble(
2423 baseIndex(heap, storage, index, m_node->child2()));
2424
2425 if (!m_node->arrayMode().isSaneChain()) {
2426 speculate(
2427 LoadFromHole, noValue(), 0,
2428 m_out.doubleNotEqualOrUnordered(result, result));
2429 }
2430 setDouble(result);
2431 break;
2432 }
2433
2434 LValue base = lowCell(m_node->child1());
2435
2436 LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetByVal double in bounds"));
2437 LBasicBlock boxPath = FTL_NEW_BLOCK(m_out, ("GetByVal double boxing"));
2438 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetByVal double slow case"));
2439 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal double continuation"));
2440
2441 m_out.branch(
2442 m_out.aboveOrEqual(
2443 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
2444 rarely(slowCase), usually(inBounds));
2445
2446 LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath);
2447 LValue doubleValue = m_out.loadDouble(
2448 baseIndex(heap, storage, index, m_node->child2()));
2449 m_out.branch(
2450 m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue),
2451 rarely(slowCase), usually(boxPath));
2452
2453 m_out.appendTo(boxPath, slowCase);
2454 ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue));
2455 m_out.jump(continuation);
2456
2457 m_out.appendTo(slowCase, continuation);
2458 ValueFromBlock slowResult = m_out.anchor(
2459 vmCall(m_out.operation(operationGetByValArrayInt), m_callFrame, base, index));
2460 m_out.jump(continuation);
2461
2462 m_out.appendTo(continuation, lastNext);
2463 setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2464 return;
2465 }
2466
2467 case Array::DirectArguments: {
2468 LValue base = lowCell(m_node->child1());
2469 LValue index = lowInt32(m_node->child2());
2470
2471 speculate(
2472 ExoticObjectMode, noValue(), nullptr,
2473 m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_overrides)));
2474 speculate(
2475 ExoticObjectMode, noValue(), nullptr,
2476 m_out.aboveOrEqual(
2477 index,
2478 m_out.load32NonNegative(base, m_heaps.DirectArguments_length)));
2479
2480 TypedPointer address = m_out.baseIndex(
2481 m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index));
2482 setJSValue(m_out.load64(address));
2483 return;
2484 }
2485
2486 case Array::ScopedArguments: {
2487 LValue base = lowCell(m_node->child1());
2488 LValue index = lowInt32(m_node->child2());
2489
2490 speculate(
2491 ExoticObjectMode, noValue(), nullptr,
2492 m_out.aboveOrEqual(
2493 index,
2494 m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength)));
2495
2496 LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table);
2497 LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length);
2498
2499 LBasicBlock namedCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments named case"));
2500 LBasicBlock overflowCase = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments overflow case"));
2501 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal ScopedArguments continuation"));
2502
2503 m_out.branch(
2504 m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase));
2505
2506 LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase);
2507
2508 LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope);
2509 LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments);
2510
2511 TypedPointer address = m_out.baseIndex(
2512 m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index));
2513 LValue scopeOffset = m_out.load32(address);
2514
2515 speculate(
2516 ExoticObjectMode, noValue(), nullptr,
2517 m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset)));
2518
2519 address = m_out.baseIndex(
2520 m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset));
2521 ValueFromBlock namedResult = m_out.anchor(m_out.load64(address));
2522 m_out.jump(continuation);
2523
2524 m_out.appendTo(overflowCase, continuation);
2525
2526 address = m_out.baseIndex(
2527 m_heaps.ScopedArguments_overflowStorage, base,
2528 m_out.zeroExtPtr(m_out.sub(index, namedLength)));
2529 LValue overflowValue = m_out.load64(address);
2530 speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue));
2531 ValueFromBlock overflowResult = m_out.anchor(overflowValue);
2532 m_out.jump(continuation);
2533
2534 m_out.appendTo(continuation, lastNext);
2535 setJSValue(m_out.phi(m_out.int64, namedResult, overflowResult));
2536 return;
2537 }
2538
2539 case Array::Generic: {
2540 setJSValue(vmCall(
2541 m_out.operation(operationGetByVal), m_callFrame,
2542 lowJSValue(m_node->child1()), lowJSValue(m_node->child2())));
2543 return;
2544 }
2545
2546 case Array::String: {
2547 compileStringCharAt();
2548 return;
2549 }
2550
2551 default: {
2552 LValue index = lowInt32(m_node->child2());
2553 LValue storage = lowStorage(m_node->child3());
2554
2555 TypedArrayType type = m_node->arrayMode().typedArrayType();
2556
2557 if (isTypedView(type)) {
2558 TypedPointer pointer = TypedPointer(
2559 m_heaps.typedArrayProperties,
2560 m_out.add(
2561 storage,
2562 m_out.shl(
2563 m_out.zeroExtPtr(index),
2564 m_out.constIntPtr(logElementSize(type)))));
2565
2566 if (isInt(type)) {
2567 LValue result;
2568 switch (elementSize(type)) {
2569 case 1:
2570 result = m_out.load8(pointer);
2571 break;
2572 case 2:
2573 result = m_out.load16(pointer);
2574 break;
2575 case 4:
2576 result = m_out.load32(pointer);
2577 break;
2578 default:
2579 DFG_CRASH(m_graph, m_node, "Bad element size");
2580 }
2581
2582 if (elementSize(type) < 4) {
2583 if (isSigned(type))
2584 result = m_out.signExt(result, m_out.int32);
2585 else
2586 result = m_out.zeroExt(result, m_out.int32);
2587 setInt32(result);
2588 return;
2589 }
2590
2591 if (isSigned(type)) {
2592 setInt32(result);
2593 return;
2594 }
2595
2596 if (m_node->shouldSpeculateInt32()) {
2597 speculate(
2598 Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero));
2599 setInt32(result);
2600 return;
2601 }
2602
2603 if (m_node->shouldSpeculateMachineInt()) {
2604 setStrictInt52(m_out.zeroExt(result, m_out.int64));
2605 return;
2606 }
2607
2608 setDouble(m_out.unsignedToFP(result, m_out.doubleType));
2609 return;
2610 }
2611
2612 ASSERT(isFloat(type));
2613
2614 LValue result;
2615 switch (type) {
2616 case TypeFloat32:
2617 result = m_out.fpCast(m_out.loadFloat(pointer), m_out.doubleType);
2618 break;
2619 case TypeFloat64:
2620 result = m_out.loadDouble(pointer);
2621 break;
2622 default:
2623 DFG_CRASH(m_graph, m_node, "Bad typed array type");
2624 }
2625
2626 setDouble(result);
2627 return;
2628 }
2629
2630 DFG_CRASH(m_graph, m_node, "Bad array type");
2631 return;
2632 } }
2633 }
2634
2635 void compileGetMyArgumentByVal()
2636 {
2637 InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame;
2638
2639 LValue index = lowInt32(m_node->child2());
2640
2641 LValue limit;
2642 if (inlineCallFrame && !inlineCallFrame->isVarargs())
2643 limit = m_out.constInt32(inlineCallFrame->arguments.size() - 1);
2644 else {
2645 VirtualRegister argumentCountRegister;
2646 if (!inlineCallFrame)
2647 argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
2648 else
2649 argumentCountRegister = inlineCallFrame->argumentCountRegister;
2650 limit = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
2651 }
2652
2653 speculate(ExoticObjectMode, noValue(), 0, m_out.aboveOrEqual(index, limit));
2654
2655 TypedPointer base;
2656 if (inlineCallFrame) {
2657 if (inlineCallFrame->arguments.size() <= 1) {
2658 // We should have already exited due to the bounds check, above. Just tell the
2659 // compiler that anything dominated by this instruction is not reachable, so
2660 // that we don't waste time generating such code. This will also plant some
2661 // kind of crashing instruction so that if by some fluke the bounds check didn't
2662 // work, we'll crash in an easy-to-see way.
2663 didAlreadyTerminate();
2664 return;
2665 }
2666 base = addressFor(inlineCallFrame->arguments[1].virtualRegister());
2667 } else
2668 base = addressFor(virtualRegisterForArgument(1));
2669
2670 LValue pointer = m_out.baseIndex(
2671 base.value(), m_out.zeroExt(index, m_out.intPtr), ScaleEight);
2672 setJSValue(m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)));
2673 }
2674
2675 void compilePutByVal()
2676 {
2677 Edge child1 = m_graph.varArgChild(m_node, 0);
2678 Edge child2 = m_graph.varArgChild(m_node, 1);
2679 Edge child3 = m_graph.varArgChild(m_node, 2);
2680 Edge child4 = m_graph.varArgChild(m_node, 3);
2681 Edge child5 = m_graph.varArgChild(m_node, 4);
2682
2683 switch (m_node->arrayMode().type()) {
2684 case Array::Generic: {
2685 V_JITOperation_EJJJ operation;
2686 if (m_node->op() == PutByValDirect) {
2687 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2688 operation = operationPutByValDirectStrict;
2689 else
2690 operation = operationPutByValDirectNonStrict;
2691 } else {
2692 if (m_graph.isStrictModeFor(m_node->origin.semantic))
2693 operation = operationPutByValStrict;
2694 else
2695 operation = operationPutByValNonStrict;
2696 }
2697
2698 vmCall(
2699 m_out.operation(operation), m_callFrame,
2700 lowJSValue(child1), lowJSValue(child2), lowJSValue(child3));
2701 return;
2702 }
2703
2704 default:
2705 break;
2706 }
2707
2708 LValue base = lowCell(child1);
2709 LValue index = lowInt32(child2);
2710 LValue storage = lowStorage(child4);
2711
2712 switch (m_node->arrayMode().type()) {
2713 case Array::Int32:
2714 case Array::Double:
2715 case Array::Contiguous: {
2716 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal continuation"));
2717 LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation);
2718
2719 switch (m_node->arrayMode().type()) {
2720 case Array::Int32:
2721 case Array::Contiguous: {
2722 LValue value = lowJSValue(child3, ManualOperandSpeculation);
2723
2724 if (m_node->arrayMode().type() == Array::Int32)
2725 FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32, isNotInt32(value));
2726
2727 TypedPointer elementPointer = m_out.baseIndex(
2728 m_node->arrayMode().type() == Array::Int32 ?
2729 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties,
2730 storage, m_out.zeroExtPtr(index), provenValue(child2));
2731
2732 if (m_node->op() == PutByValAlias) {
2733 m_out.store64(value, elementPointer);
2734 break;
2735 }
2736
2737 contiguousPutByValOutOfBounds(
2738 codeBlock()->isStrictMode()
2739 ? operationPutByValBeyondArrayBoundsStrict
2740 : operationPutByValBeyondArrayBoundsNonStrict,
2741 base, storage, index, value, continuation);
2742
2743 m_out.store64(value, elementPointer);
2744 break;
2745 }
2746
2747 case Array::Double: {
2748 LValue value = lowDouble(child3);
2749
2750 FTL_TYPE_CHECK(
2751 doubleValue(value), child3, SpecDoubleReal,
2752 m_out.doubleNotEqualOrUnordered(value, value));
2753
2754 TypedPointer elementPointer = m_out.baseIndex(
2755 m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index),
2756 provenValue(child2));
2757
2758 if (m_node->op() == PutByValAlias) {
2759 m_out.storeDouble(value, elementPointer);
2760 break;
2761 }
2762
2763 contiguousPutByValOutOfBounds(
2764 codeBlock()->isStrictMode()
2765 ? operationPutDoubleByValBeyondArrayBoundsStrict
2766 : operationPutDoubleByValBeyondArrayBoundsNonStrict,
2767 base, storage, index, value, continuation);
2768
2769 m_out.storeDouble(value, elementPointer);
2770 break;
2771 }
2772
2773 default:
2774 DFG_CRASH(m_graph, m_node, "Bad array type");
2775 }
2776
2777 m_out.jump(continuation);
2778 m_out.appendTo(continuation, outerLastNext);
2779 return;
2780 }
2781
2782 default:
2783 TypedArrayType type = m_node->arrayMode().typedArrayType();
2784
2785 if (isTypedView(type)) {
2786 TypedPointer pointer = TypedPointer(
2787 m_heaps.typedArrayProperties,
2788 m_out.add(
2789 storage,
2790 m_out.shl(
2791 m_out.zeroExt(index, m_out.intPtr),
2792 m_out.constIntPtr(logElementSize(type)))));
2793
2794 LType refType;
2795 LValue valueToStore;
2796
2797 if (isInt(type)) {
2798 LValue intValue;
2799 switch (child3.useKind()) {
2800 case Int52RepUse:
2801 case Int32Use: {
2802 if (child3.useKind() == Int32Use)
2803 intValue = lowInt32(child3);
2804 else
2805 intValue = m_out.castToInt32(lowStrictInt52(child3));
2806
2807 if (isClamped(type)) {
2808 ASSERT(elementSize(type) == 1);
2809
2810 LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp atLeastZero"));
2811 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal int clamp continuation"));
2812
2813 Vector<ValueFromBlock, 2> intValues;
2814 intValues.append(m_out.anchor(m_out.int32Zero));
2815 m_out.branch(
2816 m_out.lessThan(intValue, m_out.int32Zero),
2817 unsure(continuation), unsure(atLeastZero));
2818
2819 LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation);
2820
2821 intValues.append(m_out.anchor(m_out.select(
2822 m_out.greaterThan(intValue, m_out.constInt32(255)),
2823 m_out.constInt32(255),
2824 intValue)));
2825 m_out.jump(continuation);
2826
2827 m_out.appendTo(continuation, lastNext);
2828 intValue = m_out.phi(m_out.int32, intValues);
2829 }
2830 break;
2831 }
2832
2833 case DoubleRepUse: {
2834 LValue doubleValue = lowDouble(child3);
2835
2836 if (isClamped(type)) {
2837 ASSERT(elementSize(type) == 1);
2838
2839 LBasicBlock atLeastZero = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp atLeastZero"));
2840 LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp withinRange"));
2841 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal double clamp continuation"));
2842
2843 Vector<ValueFromBlock, 3> intValues;
2844 intValues.append(m_out.anchor(m_out.int32Zero));
2845 m_out.branch(
2846 m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero),
2847 unsure(continuation), unsure(atLeastZero));
2848
2849 LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange);
2850 intValues.append(m_out.anchor(m_out.constInt32(255)));
2851 m_out.branch(
2852 m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)),
2853 unsure(continuation), unsure(withinRange));
2854
2855 m_out.appendTo(withinRange, continuation);
2856 intValues.append(m_out.anchor(m_out.fpToInt32(doubleValue)));
2857 m_out.jump(continuation);
2858
2859 m_out.appendTo(continuation, lastNext);
2860 intValue = m_out.phi(m_out.int32, intValues);
2861 } else
2862 intValue = doubleToInt32(doubleValue);
2863 break;
2864 }
2865
2866 default:
2867 DFG_CRASH(m_graph, m_node, "Bad use kind");
2868 }
2869
2870 switch (elementSize(type)) {
2871 case 1:
2872 valueToStore = m_out.intCast(intValue, m_out.int8);
2873 refType = m_out.ref8;
2874 break;
2875 case 2:
2876 valueToStore = m_out.intCast(intValue, m_out.int16);
2877 refType = m_out.ref16;
2878 break;
2879 case 4:
2880 valueToStore = intValue;
2881 refType = m_out.ref32;
2882 break;
2883 default:
2884 DFG_CRASH(m_graph, m_node, "Bad element size");
2885 }
2886 } else /* !isInt(type) */ {
2887 LValue value = lowDouble(child3);
2888 switch (type) {
2889 case TypeFloat32:
2890 valueToStore = m_out.fpCast(value, m_out.floatType);
2891 refType = m_out.refFloat;
2892 break;
2893 case TypeFloat64:
2894 valueToStore = value;
2895 refType = m_out.refDouble;
2896 break;
2897 default:
2898 DFG_CRASH(m_graph, m_node, "Bad typed array type");
2899 }
2900 }
2901
2902 if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias)
2903 m_out.store(valueToStore, pointer, refType);
2904 else {
2905 LBasicBlock isInBounds = FTL_NEW_BLOCK(m_out, ("PutByVal typed array in bounds case"));
2906 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("PutByVal typed array continuation"));
2907
2908 m_out.branch(
2909 m_out.aboveOrEqual(index, lowInt32(child5)),
2910 unsure(continuation), unsure(isInBounds));
2911
2912 LBasicBlock lastNext = m_out.appendTo(isInBounds, continuation);
2913 m_out.store(valueToStore, pointer, refType);
2914 m_out.jump(continuation);
2915
2916 m_out.appendTo(continuation, lastNext);
2917 }
2918
2919 return;
2920 }
2921
2922 DFG_CRASH(m_graph, m_node, "Bad array type");
2923 break;
2924 }
2925 }
2926
2927 void compileArrayPush()
2928 {
2929 LValue base = lowCell(m_node->child1());
2930 LValue storage = lowStorage(m_node->child3());
2931
2932 switch (m_node->arrayMode().type()) {
2933 case Array::Int32:
2934 case Array::Contiguous:
2935 case Array::Double: {
2936 LValue value;
2937 LType refType;
2938
2939 if (m_node->arrayMode().type() != Array::Double) {
2940 value = lowJSValue(m_node->child2(), ManualOperandSpeculation);
2941 if (m_node->arrayMode().type() == Array::Int32) {
2942 FTL_TYPE_CHECK(
2943 jsValueValue(value), m_node->child2(), SpecInt32, isNotInt32(value));
2944 }
2945 refType = m_out.ref64;
2946 } else {
2947 value = lowDouble(m_node->child2());
2948 FTL_TYPE_CHECK(
2949 doubleValue(value), m_node->child2(), SpecDoubleReal,
2950 m_out.doubleNotEqualOrUnordered(value, value));
2951 refType = m_out.refDouble;
2952 }
2953
2954 IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
2955
2956 LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
2957
2958 LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("ArrayPush fast path"));
2959 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("ArrayPush slow path"));
2960 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPush continuation"));
2961
2962 m_out.branch(
2963 m_out.aboveOrEqual(
2964 prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)),
2965 rarely(slowPath), usually(fastPath));
2966
2967 LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
2968 m_out.store(
2969 value, m_out.baseIndex(heap, storage, m_out.zeroExtPtr(prevLength)), refType);
2970 LValue newLength = m_out.add(prevLength, m_out.int32One);
2971 m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
2972
2973 ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength));
2974 m_out.jump(continuation);
2975
2976 m_out.appendTo(slowPath, continuation);
2977 LValue operation;
2978 if (m_node->arrayMode().type() != Array::Double)
2979 operation = m_out.operation(operationArrayPush);
2980 else
2981 operation = m_out.operation(operationArrayPushDouble);
2982 ValueFromBlock slowResult = m_out.anchor(
2983 vmCall(operation, m_callFrame, value, base));
2984 m_out.jump(continuation);
2985
2986 m_out.appendTo(continuation, lastNext);
2987 setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
2988 return;
2989 }
2990
2991 default:
2992 DFG_CRASH(m_graph, m_node, "Bad array type");
2993 return;
2994 }
2995 }
2996
2997 void compileArrayPop()
2998 {
2999 LValue base = lowCell(m_node->child1());
3000 LValue storage = lowStorage(m_node->child2());
3001
3002 switch (m_node->arrayMode().type()) {
3003 case Array::Int32:
3004 case Array::Double:
3005 case Array::Contiguous: {
3006 IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type());
3007
3008 LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("ArrayPop fast case"));
3009 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("ArrayPop slow case"));
3010 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ArrayPop continuation"));
3011
3012 LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength);
3013
3014 Vector<ValueFromBlock, 3> results;
3015 results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
3016 m_out.branch(
3017 m_out.isZero32(prevLength), rarely(continuation), usually(fastCase));
3018
3019 LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase);
3020 LValue newLength = m_out.sub(prevLength, m_out.int32One);
3021 m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength);
3022 TypedPointer pointer = m_out.baseIndex(heap, storage, m_out.zeroExtPtr(newLength));
3023 if (m_node->arrayMode().type() != Array::Double) {
3024 LValue result = m_out.load64(pointer);
3025 m_out.store64(m_out.int64Zero, pointer);
3026 results.append(m_out.anchor(result));
3027 m_out.branch(
3028 m_out.notZero64(result), usually(continuation), rarely(slowCase));
3029 } else {
3030 LValue result = m_out.loadDouble(pointer);
3031 m_out.store64(m_out.constInt64(bitwise_cast<int64_t>(PNaN)), pointer);
3032 results.append(m_out.anchor(boxDouble(result)));
3033 m_out.branch(
3034 m_out.doubleEqual(result, result),
3035 usually(continuation), rarely(slowCase));
3036 }
3037
3038 m_out.appendTo(slowCase, continuation);
3039 results.append(m_out.anchor(vmCall(
3040 m_out.operation(operationArrayPopAndRecoverLength), m_callFrame, base)));
3041 m_out.jump(continuation);
3042
3043 m_out.appendTo(continuation, lastNext);
3044 setJSValue(m_out.phi(m_out.int64, results));
3045 return;
3046 }
3047
3048 default:
3049 DFG_CRASH(m_graph, m_node, "Bad array type");
3050 return;
3051 }
3052 }
3053
3054 void compileCreateActivation()
3055 {
3056 LValue scope = lowCell(m_node->child1());
3057 SymbolTable* table = m_node->castOperand<SymbolTable*>();
3058 Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure();
3059
3060 if (table->singletonScope()->isStillValid()) {
3061 LValue callResult = vmCall(
3062 m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
3063 scope, weakPointer(table));
3064 setJSValue(callResult);
3065 return;
3066 }
3067
3068 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateActivation slow path"));
3069 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateActivation continuation"));
3070
3071 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3072
3073 LValue fastObject = allocateObject<JSLexicalEnvironment>(
3074 JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath);
3075
3076 // We don't need memory barriers since we just fast-created the activation, so the
3077 // activation must be young.
3078 m_out.storePtr(scope, fastObject, m_heaps.JSScope_next);
3079 m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable);
3080
3081 for (unsigned i = 0; i < table->scopeSize(); ++i) {
3082 m_out.store64(
3083 m_out.constInt64(JSValue::encode(jsUndefined())),
3084 fastObject, m_heaps.JSEnvironmentRecord_variables[i]);
3085 }
3086
3087 ValueFromBlock fastResult = m_out.anchor(fastObject);
3088 m_out.jump(continuation);
3089
3090 m_out.appendTo(slowPath, continuation);
3091 LValue callResult = vmCall(
3092 m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
3093 scope, weakPointer(table));
3094 ValueFromBlock slowResult = m_out.anchor(callResult);
3095 m_out.jump(continuation);
3096
3097 m_out.appendTo(continuation, lastNext);
3098 setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
3099 }
3100
3101 void compileNewFunction()
3102 {
3103 LValue scope = lowCell(m_node->child1());
3104 FunctionExecutable* executable = m_node->castOperand<FunctionExecutable*>();
3105 if (executable->singletonFunction()->isStillValid()) {
3106 LValue callResult = vmCall(
3107 m_out.operation(operationNewFunction), m_callFrame, scope, weakPointer(executable));
3108 setJSValue(callResult);
3109 return;
3110 }
3111
3112 Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->functionStructure();
3113
3114 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("NewFunction slow path"));
3115 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewFunction continuation"));
3116
3117 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3118
3119 LValue fastObject = allocateObject<JSFunction>(
3120 structure, m_out.intPtrZero, slowPath);
3121
3122 // We don't need memory barriers since we just fast-created the function, so it
3123 // must be young.
3124 m_out.storePtr(scope, fastObject, m_heaps.JSFunction_scope);
3125 m_out.storePtr(weakPointer(executable), fastObject, m_heaps.JSFunction_executable);
3126 m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.JSFunction_rareData);
3127
3128 ValueFromBlock fastResult = m_out.anchor(fastObject);
3129 m_out.jump(continuation);
3130
3131 m_out.appendTo(slowPath, continuation);
3132 LValue callResult = vmCall(
3133 m_out.operation(operationNewFunctionWithInvalidatedReallocationWatchpoint),
3134 m_callFrame, scope, weakPointer(executable));
3135 ValueFromBlock slowResult = m_out.anchor(callResult);
3136 m_out.jump(continuation);
3137
3138 m_out.appendTo(continuation, lastNext);
3139 setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
3140 }
3141
3142 void compileCreateDirectArguments()
3143 {
3144 // FIXME: A more effective way of dealing with the argument count and callee is to have
3145 // them be explicit arguments to this node.
3146 // https://bugs.webkit.org/show_bug.cgi?id=142207
3147
3148 Structure* structure =
3149 m_graph.globalObjectFor(m_node->origin.semantic)->directArgumentsStructure();
3150
3151 unsigned minCapacity = m_graph.baselineCodeBlockFor(m_node->origin.semantic)->numParameters() - 1;
3152
3153 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments slow path"));
3154 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments continuation"));
3155
3156 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3157
3158 ArgumentsLength length = getArgumentsLength();
3159
3160 LValue fastObject;
3161 if (length.isKnown) {
3162 fastObject = allocateObject<DirectArguments>(
3163 DirectArguments::allocationSize(std::max(length.known, minCapacity)), structure,
3164 m_out.intPtrZero, slowPath);
3165 } else {
3166 LValue size = m_out.add(
3167 m_out.shl(length.value, m_out.constInt32(3)),
3168 m_out.constInt32(DirectArguments::storageOffset()));
3169
3170 size = m_out.select(
3171 m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)),
3172 size, m_out.constInt32(DirectArguments::allocationSize(minCapacity)));
3173
3174 fastObject = allocateVariableSizedObject<DirectArguments>(
3175 size, structure, m_out.intPtrZero, slowPath);
3176 }
3177
3178 m_out.store32(length.value, fastObject, m_heaps.DirectArguments_length);
3179 m_out.store32(m_out.constInt32(minCapacity), fastObject, m_heaps.DirectArguments_minCapacity);
3180 m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.DirectArguments_overrides);
3181
3182 ValueFromBlock fastResult = m_out.anchor(fastObject);
3183 m_out.jump(continuation);
3184
3185 m_out.appendTo(slowPath, continuation);
3186 LValue callResult = vmCall(
3187 m_out.operation(operationCreateDirectArguments), m_callFrame, weakPointer(structure),
3188 length.value, m_out.constInt32(minCapacity));
3189 ValueFromBlock slowResult = m_out.anchor(callResult);
3190 m_out.jump(continuation);
3191
3192 m_out.appendTo(continuation, lastNext);
3193 LValue result = m_out.phi(m_out.intPtr, fastResult, slowResult);
3194
3195 m_out.storePtr(getCurrentCallee(), result, m_heaps.DirectArguments_callee);
3196
3197 if (length.isKnown) {
3198 VirtualRegister start = AssemblyHelpers::argumentsStart(m_node->origin.semantic);
3199 for (unsigned i = 0; i < std::max(length.known, minCapacity); ++i) {
3200 m_out.store64(
3201 m_out.load64(addressFor(start + i)),
3202 result, m_heaps.DirectArguments_storage[i]);
3203 }
3204 } else {
3205 LValue stackBase = getArgumentsStart();
3206
3207 LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments loop body"));
3208 LBasicBlock end = FTL_NEW_BLOCK(m_out, ("CreateDirectArguments loop end"));
3209
3210 ValueFromBlock originalLength;
3211 if (minCapacity) {
3212 LValue capacity = m_out.select(
3213 m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)),
3214 length.value,
3215 m_out.constInt32(minCapacity));
3216 originalLength = m_out.anchor(m_out.zeroExtPtr(capacity));
3217 m_out.jump(loop);
3218 } else {
3219 originalLength = m_out.anchor(m_out.zeroExtPtr(length.value));
3220 m_out.branch(m_out.isNull(originalLength.value()), unsure(end), unsure(loop));
3221 }
3222
3223 lastNext = m_out.appendTo(loop, end);
3224 LValue previousIndex = m_out.phi(m_out.intPtr, originalLength);
3225 LValue index = m_out.sub(previousIndex, m_out.intPtrOne);
3226 m_out.store64(
3227 m_out.load64(m_out.baseIndex(m_heaps.variables, stackBase, index)),
3228 m_out.baseIndex(m_heaps.DirectArguments_storage, result, index));
3229 ValueFromBlock nextIndex = m_out.anchor(index);
3230 addIncoming(previousIndex, nextIndex);
3231 m_out.branch(m_out.isNull(index), unsure(end), unsure(loop));
3232
3233 m_out.appendTo(end, lastNext);
3234 }
3235
3236 setJSValue(result);
3237 }
3238
3239 void compileCreateScopedArguments()
3240 {
3241 LValue scope = lowCell(m_node->child1());
3242
3243 LValue result = vmCall(
3244 m_out.operation(operationCreateScopedArguments), m_callFrame,
3245 weakPointer(
3246 m_graph.globalObjectFor(m_node->origin.semantic)->scopedArgumentsStructure()),
3247 getArgumentsStart(), getArgumentsLength().value, getCurrentCallee(), scope);
3248
3249 setJSValue(result);
3250 }
3251
3252 void compileCreateClonedArguments()
3253 {
3254 LValue result = vmCall(
3255 m_out.operation(operationCreateClonedArguments), m_callFrame,
3256 weakPointer(
3257 m_graph.globalObjectFor(m_node->origin.semantic)->outOfBandArgumentsStructure()),
3258 getArgumentsStart(), getArgumentsLength().value, getCurrentCallee());
3259
3260 setJSValue(result);
3261 }
3262
3263 void compileNewObject()
3264 {
3265 setJSValue(allocateObject(m_node->structure()));
3266 }
3267
3268 void compileNewArray()
3269 {
3270 // First speculate appropriately on all of the children. Do this unconditionally up here
3271 // because some of the slow paths may otherwise forget to do it. It's sort of arguable
3272 // that doing the speculations up here might be unprofitable for RA - so we can consider
3273 // sinking this to below the allocation fast path if we find that this has a lot of
3274 // register pressure.
3275 for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex)
3276 speculate(m_graph.varArgChild(m_node, operandIndex));
3277
3278 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3279 Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
3280 m_node->indexingType());
3281
3282 DFG_ASSERT(m_graph, m_node, structure->indexingType() == m_node->indexingType());
3283
3284 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
3285 unsigned numElements = m_node->numChildren();
3286
3287 ArrayValues arrayValues = allocateJSArray(structure, numElements);
3288
3289 for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
3290 Edge edge = m_graph.varArgChild(m_node, operandIndex);
3291
3292 switch (m_node->indexingType()) {
3293 case ALL_BLANK_INDEXING_TYPES:
3294 case ALL_UNDECIDED_INDEXING_TYPES:
3295 DFG_CRASH(m_graph, m_node, "Bad indexing type");
3296 break;
3297
3298 case ALL_DOUBLE_INDEXING_TYPES:
3299 m_out.storeDouble(
3300 lowDouble(edge),
3301 arrayValues.butterfly, m_heaps.indexedDoubleProperties[operandIndex]);
3302 break;
3303
3304 case ALL_INT32_INDEXING_TYPES:
3305 case ALL_CONTIGUOUS_INDEXING_TYPES:
3306 m_out.store64(
3307 lowJSValue(edge, ManualOperandSpeculation),
3308 arrayValues.butterfly,
3309 m_heaps.forIndexingType(m_node->indexingType())->at(operandIndex));
3310 break;
3311
3312 default:
3313 DFG_CRASH(m_graph, m_node, "Corrupt indexing type");
3314 break;
3315 }
3316 }
3317
3318 setJSValue(arrayValues.array);
3319 return;
3320 }
3321
3322 if (!m_node->numChildren()) {
3323 setJSValue(vmCall(
3324 m_out.operation(operationNewEmptyArray), m_callFrame,
3325 m_out.constIntPtr(structure)));
3326 return;
3327 }
3328
3329 size_t scratchSize = sizeof(EncodedJSValue) * m_node->numChildren();
3330 ASSERT(scratchSize);
3331 ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize);
3332 EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
3333
3334 for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) {
3335 Edge edge = m_graph.varArgChild(m_node, operandIndex);
3336 m_out.store64(
3337 lowJSValue(edge, ManualOperandSpeculation),
3338 m_out.absolute(buffer + operandIndex));
3339 }
3340
3341 m_out.storePtr(
3342 m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->activeLengthPtr()));
3343
3344 LValue result = vmCall(
3345 m_out.operation(operationNewArray), m_callFrame,
3346 m_out.constIntPtr(structure), m_out.constIntPtr(buffer),
3347 m_out.constIntPtr(m_node->numChildren()));
3348
3349 m_out.storePtr(m_out.intPtrZero, m_out.absolute(scratchBuffer->activeLengthPtr()));
3350
3351 setJSValue(result);
3352 }
3353
3354 void compileNewArrayBuffer()
3355 {
3356 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3357 Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
3358 m_node->indexingType());
3359
3360 DFG_ASSERT(m_graph, m_node, structure->indexingType() == m_node->indexingType());
3361
3362 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
3363 unsigned numElements = m_node->numConstants();
3364
3365 ArrayValues arrayValues = allocateJSArray(structure, numElements);
3366
3367 JSValue* data = codeBlock()->constantBuffer(m_node->startConstant());
3368 for (unsigned index = 0; index < m_node->numConstants(); ++index) {
3369 int64_t value;
3370 if (hasDouble(m_node->indexingType()))
3371 value = bitwise_cast<int64_t>(data[index].asNumber());
3372 else
3373 value = JSValue::encode(data[index]);
3374
3375 m_out.store64(
3376 m_out.constInt64(value),
3377 arrayValues.butterfly,
3378 m_heaps.forIndexingType(m_node->indexingType())->at(index));
3379 }
3380
3381 setJSValue(arrayValues.array);
3382 return;
3383 }
3384
3385 setJSValue(vmCall(
3386 m_out.operation(operationNewArrayBuffer), m_callFrame,
3387 m_out.constIntPtr(structure), m_out.constIntPtr(m_node->startConstant()),
3388 m_out.constIntPtr(m_node->numConstants())));
3389 }
3390
3391 void compileNewArrayWithSize()
3392 {
3393 LValue publicLength = lowInt32(m_node->child1());
3394
3395 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3396 Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(
3397 m_node->indexingType());
3398
3399 if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) {
3400 ASSERT(
3401 hasUndecided(structure->indexingType())
3402 || hasInt32(structure->indexingType())
3403 || hasDouble(structure->indexingType())
3404 || hasContiguous(structure->indexingType()));
3405
3406 LBasicBlock fastCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fast case"));
3407 LBasicBlock largeCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize large case"));
3408 LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize fail case"));
3409 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize slow case"));
3410 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize continuation"));
3411
3412 m_out.branch(
3413 m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)),
3414 rarely(largeCase), usually(fastCase));
3415
3416 LBasicBlock lastNext = m_out.appendTo(fastCase, largeCase);
3417
3418 // We don't round up to BASE_VECTOR_LEN for new Array(blah).
3419 LValue vectorLength = publicLength;
3420
3421 LValue payloadSize =
3422 m_out.shl(m_out.zeroExt(vectorLength, m_out.intPtr), m_out.constIntPtr(3));
3423
3424 LValue butterflySize = m_out.add(
3425 payloadSize, m_out.constIntPtr(sizeof(IndexingHeader)));
3426
3427 LValue endOfStorage = allocateBasicStorageAndGetEnd(butterflySize, failCase);
3428
3429 LValue butterfly = m_out.sub(endOfStorage, payloadSize);
3430
3431 LValue object = allocateObject<JSArray>(
3432 structure, butterfly, failCase);
3433
3434 m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength);
3435 m_out.store32(vectorLength, butterfly, m_heaps.Butterfly_vectorLength);
3436
3437 if (hasDouble(m_node->indexingType())) {
3438 LBasicBlock initLoop = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init loop"));
3439 LBasicBlock initDone = FTL_NEW_BLOCK(m_out, ("NewArrayWithSize double init done"));
3440
3441 ValueFromBlock originalIndex = m_out.anchor(vectorLength);
3442 ValueFromBlock originalPointer = m_out.anchor(butterfly);
3443 m_out.branch(
3444 m_out.notZero32(vectorLength), unsure(initLoop), unsure(initDone));
3445
3446 LBasicBlock initLastNext = m_out.appendTo(initLoop, initDone);
3447 LValue index = m_out.phi(m_out.int32, originalIndex);
3448 LValue pointer = m_out.phi(m_out.intPtr, originalPointer);
3449
3450 m_out.store64(
3451 m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
3452 TypedPointer(m_heaps.indexedDoubleProperties.atAnyIndex(), pointer));
3453
3454 LValue nextIndex = m_out.sub(index, m_out.int32One);
3455 addIncoming(index, m_out.anchor(nextIndex));
3456 addIncoming(pointer, m_out.anchor(m_out.add(pointer, m_out.intPtrEight)));
3457 m_out.branch(
3458 m_out.notZero32(nextIndex), unsure(initLoop), unsure(initDone));
3459
3460 m_out.appendTo(initDone, initLastNext);
3461 }
3462
3463 ValueFromBlock fastResult = m_out.anchor(object);
3464 m_out.jump(continuation);
3465
3466 m_out.appendTo(largeCase, failCase);
3467 ValueFromBlock largeStructure = m_out.anchor(m_out.constIntPtr(
3468 globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)));
3469 m_out.jump(slowCase);
3470
3471 m_out.appendTo(failCase, slowCase);
3472 ValueFromBlock failStructure = m_out.anchor(m_out.constIntPtr(structure));
3473 m_out.jump(slowCase);
3474
3475 m_out.appendTo(slowCase, continuation);
3476 LValue structureValue = m_out.phi(
3477 m_out.intPtr, largeStructure, failStructure);
3478 ValueFromBlock slowResult = m_out.anchor(vmCall(
3479 m_out.operation(operationNewArrayWithSize),
3480 m_callFrame, structureValue, publicLength));
3481 m_out.jump(continuation);
3482
3483 m_out.appendTo(continuation, lastNext);
3484 setJSValue(m_out.phi(m_out.intPtr, fastResult, slowResult));
3485 return;
3486 }
3487
3488 LValue structureValue = m_out.select(
3489 m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)),
3490 m_out.constIntPtr(
3491 globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)),
3492 m_out.constIntPtr(structure));
3493 setJSValue(vmCall(m_out.operation(operationNewArrayWithSize), m_callFrame, structureValue, publicLength));
3494 }
3495
3496 void compileAllocatePropertyStorage()
3497 {
3498 LValue object = lowCell(m_node->child1());
3499 setStorage(allocatePropertyStorage(object, m_node->transition()->previous));
3500 }
3501
3502 void compileReallocatePropertyStorage()
3503 {
3504 Transition* transition = m_node->transition();
3505 LValue object = lowCell(m_node->child1());
3506 LValue oldStorage = lowStorage(m_node->child2());
3507
3508 setStorage(
3509 reallocatePropertyStorage(
3510 object, oldStorage, transition->previous, transition->next));
3511 }
3512
3513 void compileToStringOrCallStringConstructor()
3514 {
3515 switch (m_node->child1().useKind()) {
3516 case StringObjectUse: {
3517 LValue cell = lowCell(m_node->child1());
3518 speculateStringObjectForCell(m_node->child1(), cell);
3519 m_interpreter.filter(m_node->child1(), SpecStringObject);
3520
3521 setJSValue(m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
3522 return;
3523 }
3524
3525 case StringOrStringObjectUse: {
3526 LValue cell = lowCell(m_node->child1());
3527 LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID);
3528
3529 LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject not string case"));
3530 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString StringOrStringObject continuation"));
3531
3532 ValueFromBlock simpleResult = m_out.anchor(cell);
3533 m_out.branch(
3534 m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())),
3535 unsure(continuation), unsure(notString));
3536
3537 LBasicBlock lastNext = m_out.appendTo(notString, continuation);
3538 speculateStringObjectForStructureID(m_node->child1(), structureID);
3539 ValueFromBlock unboxedResult = m_out.anchor(
3540 m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue));
3541 m_out.jump(continuation);
3542
3543 m_out.appendTo(continuation, lastNext);
3544 setJSValue(m_out.phi(m_out.int64, simpleResult, unboxedResult));
3545
3546 m_interpreter.filter(m_node->child1(), SpecString | SpecStringObject);
3547 return;
3548 }
3549
3550 case CellUse:
3551 case UntypedUse: {
3552 LValue value;
3553 if (m_node->child1().useKind() == CellUse)
3554 value = lowCell(m_node->child1());
3555 else
3556 value = lowJSValue(m_node->child1());
3557
3558 LBasicBlock isCell = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse is cell"));
3559 LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse not string"));
3560 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToString CellUse/UntypedUse continuation"));
3561
3562 LValue isCellPredicate;
3563 if (m_node->child1().useKind() == CellUse)
3564 isCellPredicate = m_out.booleanTrue;
3565 else
3566 isCellPredicate = this->isCell(value, provenType(m_node->child1()));
3567 m_out.branch(isCellPredicate, unsure(isCell), unsure(notString));
3568
3569 LBasicBlock lastNext = m_out.appendTo(isCell, notString);
3570 ValueFromBlock simpleResult = m_out.anchor(value);
3571 LValue isStringPredicate;
3572 if (m_node->child1()->prediction() & SpecString) {
3573 isStringPredicate = isString(value, provenType(m_node->child1()));
3574 } else
3575 isStringPredicate = m_out.booleanFalse;
3576 m_out.branch(isStringPredicate, unsure(continuation), unsure(notString));
3577
3578 m_out.appendTo(notString, continuation);
3579 LValue operation;
3580 if (m_node->child1().useKind() == CellUse)
3581 operation = m_out.operation(m_node->op() == ToString ? operationToStringOnCell : operationCallStringConstructorOnCell);
3582 else
3583 operation = m_out.operation(m_node->op() == ToString ? operationToString : operationCallStringConstructor);
3584 ValueFromBlock convertedResult = m_out.anchor(vmCall(operation, m_callFrame, value));
3585 m_out.jump(continuation);
3586
3587 m_out.appendTo(continuation, lastNext);
3588 setJSValue(m_out.phi(m_out.int64, simpleResult, convertedResult));
3589 return;
3590 }
3591
3592 default:
3593 DFG_CRASH(m_graph, m_node, "Bad use kind");
3594 break;
3595 }
3596 }
3597
3598 void compileToPrimitive()
3599 {
3600 LValue value = lowJSValue(m_node->child1());
3601
3602 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive cell case"));
3603 LBasicBlock isObjectCase = FTL_NEW_BLOCK(m_out, ("ToPrimitive object case"));
3604 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ToPrimitive continuation"));
3605
3606 Vector<ValueFromBlock, 3> results;
3607
3608 results.append(m_out.anchor(value));
3609 m_out.branch(
3610 isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
3611
3612 LBasicBlock lastNext = m_out.appendTo(isCellCase, isObjectCase);
3613 results.append(m_out.anchor(value));
3614 m_out.branch(
3615 isObject(value, provenType(m_node->child1())),
3616 unsure(isObjectCase), unsure(continuation));
3617
3618 m_out.appendTo(isObjectCase, continuation);
3619 results.append(m_out.anchor(vmCall(
3620 m_out.operation(operationToPrimitive), m_callFrame, value)));
3621 m_out.jump(continuation);
3622
3623 m_out.appendTo(continuation, lastNext);
3624 setJSValue(m_out.phi(m_out.int64, results));
3625 }
3626
3627 void compileMakeRope()
3628 {
3629 LValue kids[3];
3630 unsigned numKids;
3631 kids[0] = lowCell(m_node->child1());
3632 kids[1] = lowCell(m_node->child2());
3633 if (m_node->child3()) {
3634 kids[2] = lowCell(m_node->child3());
3635 numKids = 3;
3636 } else {
3637 kids[2] = 0;
3638 numKids = 2;
3639 }
3640
3641 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MakeRope slow path"));
3642 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MakeRope continuation"));
3643
3644 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
3645
3646 MarkedAllocator& allocator =
3647 vm().heap.allocatorForObjectWithDestructor(sizeof(JSRopeString));
3648
3649 LValue result = allocateCell(
3650 m_out.constIntPtr(&allocator),
3651 vm().stringStructure.get(),
3652 slowPath);
3653
3654 m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSString_value);
3655 for (unsigned i = 0; i < numKids; ++i)
3656 m_out.storePtr(kids[i], result, m_heaps.JSRopeString_fibers[i]);
3657 for (unsigned i = numKids; i < JSRopeString::s_maxInternalRopeLength; ++i)
3658 m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSRopeString_fibers[i]);
3659 LValue flags = m_out.load32(kids[0], m_heaps.JSString_flags);
3660 LValue length = m_out.load32(kids[0], m_heaps.JSString_length);
3661 for (unsigned i = 1; i < numKids; ++i) {
3662 flags = m_out.bitAnd(flags, m_out.load32(kids[i], m_heaps.JSString_flags));
3663 LValue lengthAndOverflow = m_out.addWithOverflow32(
3664 length, m_out.load32(kids[i], m_heaps.JSString_length));
3665 speculate(Uncountable, noValue(), 0, m_out.extractValue(lengthAndOverflow, 1));
3666 length = m_out.extractValue(lengthAndOverflow, 0);
3667 }
3668 m_out.store32(
3669 m_out.bitAnd(m_out.constInt32(JSString::Is8Bit), flags),
3670 result, m_heaps.JSString_flags);
3671 m_out.store32(length, result, m_heaps.JSString_length);
3672
3673 ValueFromBlock fastResult = m_out.anchor(result);
3674 m_out.jump(continuation);
3675
3676 m_out.appendTo(slowPath, continuation);
3677 ValueFromBlock slowResult;
3678 switch (numKids) {
3679 case 2:
3680 slowResult = m_out.anchor(vmCall(
3681 m_out.operation(operationMakeRope2), m_callFrame, kids[0], kids[1]));
3682 break;
3683 case 3:
3684 slowResult = m_out.anchor(vmCall(
3685 m_out.operation(operationMakeRope3), m_callFrame, kids[0], kids[1], kids[2]));
3686 break;
3687 default:
3688 DFG_CRASH(m_graph, m_node, "Bad number of children");
3689 break;
3690 }
3691 m_out.jump(continuation);
3692
3693 m_out.appendTo(continuation, lastNext);
3694 setJSValue(m_out.phi(m_out.int64, fastResult, slowResult));
3695 }
3696
3697 void compileStringCharAt()
3698 {
3699 LValue base = lowCell(m_node->child1());
3700 LValue index = lowInt32(m_node->child2());
3701 LValue storage = lowStorage(m_node->child3());
3702
3703 LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("GetByVal String fast path"));
3704 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("GetByVal String slow path"));
3705 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetByVal String continuation"));
3706
3707 m_out.branch(
3708 m_out.aboveOrEqual(
3709 index, m_out.load32NonNegative(base, m_heaps.JSString_length)),
3710 rarely(slowPath), usually(fastPath));
3711
3712 LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath);
3713
3714 LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
3715
3716 LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("GetByVal String 8-bit case"));
3717 LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("GetByVal String 16-bit case"));
3718 LBasicBlock bitsContinuation = FTL_NEW_BLOCK(m_out, ("GetByVal String bitness continuation"));
3719 LBasicBlock bigCharacter = FTL_NEW_BLOCK(m_out, ("GetByVal String big character"));
3720
3721 m_out.branch(
3722 m_out.testIsZero32(
3723 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
3724 m_out.constInt32(StringImpl::flagIs8Bit())),
3725 unsure(is16Bit), unsure(is8Bit));
3726
3727 m_out.appendTo(is8Bit, is16Bit);
3728
3729 ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt(
3730 m_out.load8(m_out.baseIndex(
3731 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
3732 provenValue(m_node->child2()))),
3733 m_out.int32));
3734 m_out.jump(bitsContinuation);
3735
3736 m_out.appendTo(is16Bit, bigCharacter);
3737
3738 ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt(
3739 m_out.load16(m_out.baseIndex(
3740 m_heaps.characters16, storage, m_out.zeroExtPtr(index),
3741 provenValue(m_node->child2()))),
3742 m_out.int32));
3743 m_out.branch(
3744 m_out.aboveOrEqual(char16Bit.value(), m_out.constInt32(0x100)),
3745 rarely(bigCharacter), usually(bitsContinuation));
3746
3747 m_out.appendTo(bigCharacter, bitsContinuation);
3748
3749 Vector<ValueFromBlock, 4> results;
3750 results.append(m_out.anchor(vmCall(
3751 m_out.operation(operationSingleCharacterString),
3752 m_callFrame, char16Bit.value())));
3753 m_out.jump(continuation);
3754
3755 m_out.appendTo(bitsContinuation, slowPath);
3756
3757 LValue character = m_out.phi(m_out.int32, char8Bit, char16Bit);
3758
3759 LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings());
3760
3761 results.append(m_out.anchor(m_out.loadPtr(m_out.baseIndex(
3762 m_heaps.singleCharacterStrings, smallStrings, m_out.zeroExtPtr(character)))));
3763 m_out.jump(continuation);
3764
3765 m_out.appendTo(slowPath, continuation);
3766
3767 if (m_node->arrayMode().isInBounds()) {
3768 speculate(OutOfBounds, noValue(), 0, m_out.booleanTrue);
3769 results.append(m_out.anchor(m_out.intPtrZero));
3770 } else {
3771 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
3772
3773 if (globalObject->stringPrototypeChainIsSane()) {
3774 // FIXME: This could be captured using a Speculation mode that means
3775 // "out-of-bounds loads return a trivial value", something like
3776 // SaneChainOutOfBounds.
3777 // https://bugs.webkit.org/show_bug.cgi?id=144668
3778
3779 m_graph.watchpoints().addLazily(globalObject->stringPrototype()->structure()->transitionWatchpointSet());
3780 m_graph.watchpoints().addLazily(globalObject->objectPrototype()->structure()->transitionWatchpointSet());
3781
3782 LBasicBlock negativeIndex = FTL_NEW_BLOCK(m_out, ("GetByVal String negative index"));
3783
3784 results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))));
3785 m_out.branch(
3786 m_out.lessThan(index, m_out.int32Zero),
3787 rarely(negativeIndex), usually(continuation));
3788
3789 m_out.appendTo(negativeIndex, continuation);
3790 }
3791
3792 results.append(m_out.anchor(vmCall(
3793 m_out.operation(operationGetByValStringInt), m_callFrame, base, index)));
3794 }
3795
3796 m_out.jump(continuation);
3797
3798 m_out.appendTo(continuation, lastNext);
3799 setJSValue(m_out.phi(m_out.int64, results));
3800 }
3801
3802 void compileStringCharCodeAt()
3803 {
3804 LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt 8-bit case"));
3805 LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt 16-bit case"));
3806 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("StringCharCodeAt continuation"));
3807
3808 LValue base = lowCell(m_node->child1());
3809 LValue index = lowInt32(m_node->child2());
3810 LValue storage = lowStorage(m_node->child3());
3811
3812 speculate(
3813 Uncountable, noValue(), 0,
3814 m_out.aboveOrEqual(
3815 index, m_out.load32NonNegative(base, m_heaps.JSString_length)));
3816
3817 LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value);
3818
3819 m_out.branch(
3820 m_out.testIsZero32(
3821 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
3822 m_out.constInt32(StringImpl::flagIs8Bit())),
3823 unsure(is16Bit), unsure(is8Bit));
3824
3825 LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit);
3826
3827 ValueFromBlock char8Bit = m_out.anchor(m_out.zeroExt(
3828 m_out.load8(m_out.baseIndex(
3829 m_heaps.characters8, storage, m_out.zeroExtPtr(index),
3830 provenValue(m_node->child2()))),
3831 m_out.int32));
3832 m_out.jump(continuation);
3833
3834 m_out.appendTo(is16Bit, continuation);
3835
3836 ValueFromBlock char16Bit = m_out.anchor(m_out.zeroExt(
3837 m_out.load16(m_out.baseIndex(
3838 m_heaps.characters16, storage, m_out.zeroExtPtr(index),
3839 provenValue(m_node->child2()))),
3840 m_out.int32));
3841 m_out.jump(continuation);
3842
3843 m_out.appendTo(continuation, lastNext);
3844
3845 setInt32(m_out.phi(m_out.int32, char8Bit, char16Bit));
3846 }
3847
3848 void compileGetByOffset()
3849 {
3850 StorageAccessData& data = m_node->storageAccessData();
3851
3852 setJSValue(loadProperty(
3853 lowStorage(m_node->child1()), data.identifierNumber, data.offset));
3854 }
3855
3856 void compileGetGetter()
3857 {
3858 setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_getter));
3859 }
3860
3861 void compileGetSetter()
3862 {
3863 setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_setter));
3864 }
3865
3866 void compileMultiGetByOffset()
3867 {
3868 LValue base = lowCell(m_node->child1());
3869
3870 MultiGetByOffsetData& data = m_node->multiGetByOffsetData();
3871
3872 if (data.variants.isEmpty()) {
3873 // Protect against creating a Phi function with zero inputs. LLVM doesn't like that.
3874 terminate(BadCache);
3875 return;
3876 }
3877
3878 Vector<LBasicBlock, 2> blocks(data.variants.size());
3879 for (unsigned i = data.variants.size(); i--;)
3880 blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset case ", i));
3881 LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset fail"));
3882 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiGetByOffset continuation"));
3883
3884 Vector<SwitchCase, 2> cases;
3885 StructureSet baseSet;
3886 for (unsigned i = data.variants.size(); i--;) {
3887 GetByIdVariant variant = data.variants[i];
3888 for (unsigned j = variant.structureSet().size(); j--;) {
3889 Structure* structure = variant.structureSet()[j];
3890 baseSet.add(structure);
3891 cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1)));
3892 }
3893 }
3894 m_out.switchInstruction(
3895 m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
3896
3897 LBasicBlock lastNext = m_out.m_nextBlock;
3898
3899 Vector<ValueFromBlock, 2> results;
3900 for (unsigned i = data.variants.size(); i--;) {
3901 m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit);
3902
3903 GetByIdVariant variant = data.variants[i];
3904 baseSet.merge(variant.structureSet());
3905 LValue result;
3906 JSValue constantResult;
3907 if (variant.alternateBase()) {
3908 constantResult = m_graph.tryGetConstantProperty(
3909 variant.alternateBase(), variant.baseStructure(), variant.offset());
3910 }
3911 if (constantResult)
3912 result = m_out.constInt64(JSValue::encode(constantResult));
3913 else {
3914 LValue propertyBase;
3915 if (variant.alternateBase())
3916 propertyBase = weakPointer(variant.alternateBase());
3917 else
3918 propertyBase = base;
3919 if (!isInlineOffset(variant.offset()))
3920 propertyBase = m_out.loadPtr(propertyBase, m_heaps.JSObject_butterfly);
3921 result = loadProperty(propertyBase, data.identifierNumber, variant.offset());
3922 }
3923
3924 results.append(m_out.anchor(result));
3925 m_out.jump(continuation);
3926 }
3927
3928 m_out.appendTo(exit, continuation);
3929 if (!m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet))
3930 speculate(BadCache, noValue(), nullptr, m_out.booleanTrue);
3931 m_out.unreachable();
3932
3933 m_out.appendTo(continuation, lastNext);
3934 setJSValue(m_out.phi(m_out.int64, results));
3935 }
3936
3937 void compilePutByOffset()
3938 {
3939 StorageAccessData& data = m_node->storageAccessData();
3940
3941 storeProperty(
3942 lowJSValue(m_node->child3()),
3943 lowStorage(m_node->child1()), data.identifierNumber, data.offset);
3944 }
3945
3946 void compileMultiPutByOffset()
3947 {
3948 LValue base = lowCell(m_node->child1());
3949 LValue value = lowJSValue(m_node->child2());
3950
3951 MultiPutByOffsetData& data = m_node->multiPutByOffsetData();
3952
3953 Vector<LBasicBlock, 2> blocks(data.variants.size());
3954 for (unsigned i = data.variants.size(); i--;)
3955 blocks[i] = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset case ", i));
3956 LBasicBlock exit = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset fail"));
3957 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MultiPutByOffset continuation"));
3958
3959 Vector<SwitchCase, 2> cases;
3960 StructureSet baseSet;
3961 for (unsigned i = data.variants.size(); i--;) {
3962 PutByIdVariant variant = data.variants[i];
3963 for (unsigned j = variant.oldStructure().size(); j--;) {
3964 Structure* structure = variant.oldStructure()[j];
3965 baseSet.add(structure);
3966 cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1)));
3967 }
3968 }
3969 m_out.switchInstruction(
3970 m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0));
3971
3972 LBasicBlock lastNext = m_out.m_nextBlock;
3973
3974 for (unsigned i = data.variants.size(); i--;) {
3975 m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit);
3976
3977 PutByIdVariant variant = data.variants[i];
3978
3979 LValue storage;
3980 if (variant.kind() == PutByIdVariant::Replace) {
3981 if (isInlineOffset(variant.offset()))
3982 storage = base;
3983 else
3984 storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly);
3985 } else {
3986 m_graph.m_plan.transitions.addLazily(
3987 codeBlock(), m_node->origin.semantic.codeOriginOwner(),
3988 variant.oldStructureForTransition(), variant.newStructure());
3989
3990 storage = storageForTransition(
3991 base, variant.offset(),
3992 variant.oldStructureForTransition(), variant.newStructure());
3993
3994 ASSERT(variant.oldStructureForTransition()->indexingType() == variant.newStructure()->indexingType());
3995 ASSERT(variant.oldStructureForTransition()->typeInfo().inlineTypeFlags() == variant.newStructure()->typeInfo().inlineTypeFlags());
3996 ASSERT(variant.oldStructureForTransition()->typeInfo().type() == variant.newStructure()->typeInfo().type());
3997 m_out.store32(
3998 weakStructureID(variant.newStructure()), base, m_heaps.JSCell_structureID);
3999 }
4000
4001 storeProperty(value, storage, data.identifierNumber, variant.offset());
4002 m_out.jump(continuation);
4003 }
4004
4005 m_out.appendTo(exit, continuation);
4006 if (!m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet))
4007 speculate(BadCache, noValue(), nullptr, m_out.booleanTrue);
4008 m_out.unreachable();
4009
4010 m_out.appendTo(continuation, lastNext);
4011 }
4012
4013 void compileGetGlobalVar()
4014 {
4015 setJSValue(m_out.load64(m_out.absolute(m_node->variablePointer())));
4016 }
4017
4018 void compilePutGlobalVar()
4019 {
4020 m_out.store64(
4021 lowJSValue(m_node->child2()), m_out.absolute(m_node->variablePointer()));
4022 }
4023
4024 void compileNotifyWrite()
4025 {
4026 WatchpointSet* set = m_node->watchpointSet();
4027
4028 LBasicBlock isNotInvalidated = FTL_NEW_BLOCK(m_out, ("NotifyWrite not invalidated case"));
4029 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("NotifyWrite continuation"));
4030
4031 LValue state = m_out.load8(m_out.absolute(set->addressOfState()));
4032 m_out.branch(
4033 m_out.equal(state, m_out.constInt8(IsInvalidated)),
4034 usually(continuation), rarely(isNotInvalidated));
4035
4036 LBasicBlock lastNext = m_out.appendTo(isNotInvalidated, continuation);
4037
4038 vmCall(m_out.operation(operationNotifyWrite), m_callFrame, m_out.constIntPtr(set));
4039 m_out.jump(continuation);
4040
4041 m_out.appendTo(continuation, lastNext);
4042 }
4043
4044 void compileGetCallee()
4045 {
4046 setJSValue(m_out.loadPtr(addressFor(JSStack::Callee)));
4047 }
4048
4049 void compileGetArgumentCount()
4050 {
4051 setInt32(m_out.load32(payloadFor(JSStack::ArgumentCount)));
4052 }
4053
4054 void compileGetScope()
4055 {
4056 setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSFunction_scope));
4057 }
4058
4059 void compileSkipScope()
4060 {
4061 setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next));
4062 }
4063
4064 void compileGetClosureVar()
4065 {
4066 setJSValue(
4067 m_out.load64(
4068 lowCell(m_node->child1()),
4069 m_heaps.JSEnvironmentRecord_variables[m_node->scopeOffset().offset()]));
4070 }
4071
4072 void compilePutClosureVar()
4073 {
4074 m_out.store64(
4075 lowJSValue(m_node->child2()),
4076 lowCell(m_node->child1()),
4077 m_heaps.JSEnvironmentRecord_variables[m_node->scopeOffset().offset()]);
4078 }
4079
4080 void compileGetFromArguments()
4081 {
4082 setJSValue(
4083 m_out.load64(
4084 lowCell(m_node->child1()),
4085 m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()]));
4086 }
4087
4088 void compilePutToArguments()
4089 {
4090 m_out.store64(
4091 lowJSValue(m_node->child2()),
4092 lowCell(m_node->child1()),
4093 m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()]);
4094 }
4095
4096 void compileCompareEq()
4097 {
4098 if (m_node->isBinaryUseKind(Int32Use)
4099 || m_node->isBinaryUseKind(Int52RepUse)
4100 || m_node->isBinaryUseKind(DoubleRepUse)
4101 || m_node->isBinaryUseKind(ObjectUse)
4102 || m_node->isBinaryUseKind(BooleanUse)
4103 || m_node->isBinaryUseKind(StringIdentUse)) {
4104 compileCompareStrictEq();
4105 return;
4106 }
4107
4108 if (m_node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse)) {
4109 compareEqObjectOrOtherToObject(m_node->child2(), m_node->child1());
4110 return;
4111 }
4112
4113 if (m_node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse)) {
4114 compareEqObjectOrOtherToObject(m_node->child1(), m_node->child2());
4115 return;
4116 }
4117
4118 if (m_node->isBinaryUseKind(UntypedUse)) {
4119 nonSpeculativeCompare(LLVMIntEQ, operationCompareEq);
4120 return;
4121 }
4122
4123 DFG_CRASH(m_graph, m_node, "Bad use kinds");
4124 }
4125
4126 void compileCompareEqConstant()
4127 {
4128 ASSERT(m_node->child2()->asJSValue().isNull());
4129 setBoolean(
4130 equalNullOrUndefined(
4131 m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined));
4132 }
4133
4134 void compileCompareStrictEq()
4135 {
4136 if (m_node->isBinaryUseKind(Int32Use)) {
4137 setBoolean(
4138 m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2())));
4139 return;
4140 }
4141
4142 if (m_node->isBinaryUseKind(Int52RepUse)) {
4143 Int52Kind kind;
4144 LValue left = lowWhicheverInt52(m_node->child1(), kind);
4145 LValue right = lowInt52(m_node->child2(), kind);
4146 setBoolean(m_out.equal(left, right));
4147 return;
4148 }
4149
4150 if (m_node->isBinaryUseKind(DoubleRepUse)) {
4151 setBoolean(
4152 m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2())));
4153 return;
4154 }
4155
4156 if (m_node->isBinaryUseKind(StringIdentUse)) {
4157 setBoolean(
4158 m_out.equal(lowStringIdent(m_node->child1()), lowStringIdent(m_node->child2())));
4159 return;
4160 }
4161
4162 if (m_node->isBinaryUseKind(ObjectUse, UntypedUse)) {
4163 setBoolean(
4164 m_out.equal(
4165 lowNonNullObject(m_node->child1()),
4166 lowJSValue(m_node->child2())));
4167 return;
4168 }
4169
4170 if (m_node->isBinaryUseKind(UntypedUse, ObjectUse)) {
4171 setBoolean(
4172 m_out.equal(
4173 lowNonNullObject(m_node->child2()),
4174 lowJSValue(m_node->child1())));
4175 return;
4176 }
4177
4178 if (m_node->isBinaryUseKind(ObjectUse)) {
4179 setBoolean(
4180 m_out.equal(
4181 lowNonNullObject(m_node->child1()),
4182 lowNonNullObject(m_node->child2())));
4183 return;
4184 }
4185
4186 if (m_node->isBinaryUseKind(BooleanUse)) {
4187 setBoolean(
4188 m_out.equal(lowBoolean(m_node->child1()), lowBoolean(m_node->child2())));
4189 return;
4190 }
4191
4192 if (m_node->isBinaryUseKind(MiscUse, UntypedUse)
4193 || m_node->isBinaryUseKind(UntypedUse, MiscUse)) {
4194 speculate(m_node->child1());
4195 speculate(m_node->child2());
4196 LValue left = lowJSValue(m_node->child1(), ManualOperandSpeculation);
4197 LValue right = lowJSValue(m_node->child2(), ManualOperandSpeculation);
4198 setBoolean(m_out.equal(left, right));
4199 return;
4200 }
4201
4202 if (m_node->isBinaryUseKind(StringIdentUse, NotStringVarUse)
4203 || m_node->isBinaryUseKind(NotStringVarUse, StringIdentUse)) {
4204 Edge leftEdge = m_node->childFor(StringIdentUse);
4205 Edge rightEdge = m_node->childFor(NotStringVarUse);
4206
4207 LValue left = lowStringIdent(leftEdge);
4208 LValue rightValue = lowJSValue(rightEdge, ManualOperandSpeculation);
4209
4210 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is cell case"));
4211 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar is string case"));
4212 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareStrictEq StringIdent to NotStringVar continuation"));
4213
4214 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4215 m_out.branch(
4216 isCell(rightValue, provenType(rightEdge)),
4217 unsure(isCellCase), unsure(continuation));
4218
4219 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
4220 ValueFromBlock notStringResult = m_out.anchor(m_out.booleanFalse);
4221 m_out.branch(
4222 isString(rightValue, provenType(rightEdge)),
4223 unsure(isStringCase), unsure(continuation));
4224
4225 m_out.appendTo(isStringCase, continuation);
4226 LValue right = m_out.loadPtr(rightValue, m_heaps.JSString_value);
4227 speculateStringIdent(rightEdge, rightValue, right);
4228 ValueFromBlock isStringResult = m_out.anchor(m_out.equal(left, right));
4229 m_out.jump(continuation);
4230
4231 m_out.appendTo(continuation, lastNext);
4232 setBoolean(m_out.phi(m_out.boolean, notCellResult, notStringResult, isStringResult));
4233 return;
4234 }
4235
4236 DFG_CRASH(m_graph, m_node, "Bad use kinds");
4237 }
4238
4239 void compileCompareStrictEqConstant()
4240 {
4241 JSValue constant = m_node->child2()->asJSValue();
4242
4243 setBoolean(
4244 m_out.equal(
4245 lowJSValue(m_node->child1()),
4246 m_out.constInt64(JSValue::encode(constant))));
4247 }
4248
4249 void compileCompareLess()
4250 {
4251 compare(LLVMIntSLT, LLVMRealOLT, operationCompareLess);
4252 }
4253
4254 void compileCompareLessEq()
4255 {
4256 compare(LLVMIntSLE, LLVMRealOLE, operationCompareLessEq);
4257 }
4258
4259 void compileCompareGreater()
4260 {
4261 compare(LLVMIntSGT, LLVMRealOGT, operationCompareGreater);
4262 }
4263
4264 void compileCompareGreaterEq()
4265 {
4266 compare(LLVMIntSGE, LLVMRealOGE, operationCompareGreaterEq);
4267 }
4268
4269 void compileLogicalNot()
4270 {
4271 setBoolean(m_out.bitNot(boolify(m_node->child1())));
4272 }
4273#if ENABLE(FTL_NATIVE_CALL_INLINING)
4274 void compileNativeCallOrConstruct()
4275 {
4276 int numPassedArgs = m_node->numChildren() - 1;
4277 int numArgs = numPassedArgs;
4278
4279 JSFunction* knownFunction = m_node->castOperand<JSFunction*>();
4280 NativeFunction function = knownFunction->nativeFunction();
4281
4282 Dl_info info;
4283 if (!dladdr((void*)function, &info))
4284 ASSERT(false); // if we couldn't find the native function this doesn't bode well.
4285
4286 LValue callee = getFunctionBySymbol(info.dli_sname);
4287
4288 bool notInlinable;
4289 if ((notInlinable = !callee))
4290 callee = m_out.operation(function);
4291
4292 m_out.storePtr(m_callFrame, m_execStorage, m_heaps.CallFrame_callerFrame);
4293 m_out.storePtr(constNull(m_out.intPtr), addressFor(m_execStorage, JSStack::CodeBlock));
4294 m_out.storePtr(weakPointer(knownFunction), addressFor(m_execStorage, JSStack::Callee));
4295
4296 m_out.store64(m_out.constInt64(numArgs), addressFor(m_execStorage, JSStack::ArgumentCount));
4297
4298 for (int i = 0; i < numPassedArgs; ++i) {
4299 m_out.storePtr(lowJSValue(m_graph.varArgChild(m_node, 1 + i)),
4300 addressFor(m_execStorage, JSStack::ThisArgument, i * sizeof(Register)));
4301 }
4302
4303 LValue calleeCallFrame = m_out.address(m_execState, m_heaps.CallFrame_callerFrame).value();
4304 m_out.storePtr(m_out.ptrToInt(calleeCallFrame, m_out.intPtr), m_out.absolute(&vm().topCallFrame));
4305
4306 LType typeCalleeArg;
4307 getParamTypes(getElementType(typeOf(callee)), &typeCalleeArg);
4308
4309 LValue argument = notInlinable
4310 ? m_out.ptrToInt(calleeCallFrame, typeCalleeArg)
4311 : m_out.bitCast(calleeCallFrame, typeCalleeArg);
4312 LValue call = vmCall(callee, argument);
4313
4314 if (verboseCompilationEnabled())
4315 dataLog("Native calling: ", info.dli_sname, "\n");
4316
4317 setJSValue(call);
4318 }
4319#endif
4320
4321 void compileCallOrConstruct()
4322 {
4323 int numPassedArgs = m_node->numChildren() - 1;
4324 int numArgs = numPassedArgs;
4325
4326 LValue jsCallee = lowJSValue(m_graph.varArgChild(m_node, 0));
4327
4328 unsigned stackmapID = m_stackmapIDs++;
4329
4330 Vector<LValue> arguments;
4331 arguments.append(m_out.constInt64(stackmapID));
4332 arguments.append(m_out.constInt32(sizeOfCall()));
4333 arguments.append(constNull(m_out.ref8));
4334 arguments.append(m_out.constInt32(1 + JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + numArgs));
4335 arguments.append(jsCallee); // callee -> %rax
4336 arguments.append(getUndef(m_out.int64)); // code block
4337 arguments.append(jsCallee); // callee -> stack
4338 arguments.append(m_out.constInt64(numArgs)); // argument count and zeros for the tag
4339 for (int i = 0; i < numPassedArgs; ++i)
4340 arguments.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
4341
4342 callPreflight();
4343
4344 LValue call = m_out.call(m_out.patchpointInt64Intrinsic(), arguments);
4345 setInstructionCallingConvention(call, LLVMWebKitJSCallConv);
4346
4347 m_ftlState.jsCalls.append(JSCall(stackmapID, m_node));
4348
4349 setJSValue(call);
4350 }
4351
4352 void compileCallOrConstructVarargs()
4353 {
4354 LValue jsCallee = lowJSValue(m_node->child1());
4355 LValue thisArg = lowJSValue(m_node->child3());
4356
4357 LValue jsArguments = nullptr;
4358
4359 switch (m_node->op()) {
4360 case CallVarargs:
4361 case ConstructVarargs:
4362 jsArguments = lowJSValue(m_node->child2());
4363 break;
4364 case CallForwardVarargs:
4365 case ConstructForwardVarargs:
4366 break;
4367 default:
4368 DFG_CRASH(m_graph, m_node, "bad node type");
4369 break;
4370 }
4371
4372 unsigned stackmapID = m_stackmapIDs++;
4373
4374 Vector<LValue> arguments;
4375 arguments.append(m_out.constInt64(stackmapID));
4376 arguments.append(m_out.constInt32(sizeOfICFor(m_node)));
4377 arguments.append(constNull(m_out.ref8));
4378 arguments.append(m_out.constInt32(2 + !!jsArguments));
4379 arguments.append(jsCallee);
4380 if (jsArguments)
4381 arguments.append(jsArguments);
4382 ASSERT(thisArg);
4383 arguments.append(thisArg);
4384
4385 callPreflight();
4386
4387 LValue call = m_out.call(m_out.patchpointInt64Intrinsic(), arguments);
4388 setInstructionCallingConvention(call, LLVMCCallConv);
4389
4390 m_ftlState.jsCallVarargses.append(JSCallVarargs(stackmapID, m_node));
4391
4392 setJSValue(call);
4393 }
4394
4395 void compileLoadVarargs()
4396 {
4397 LoadVarargsData* data = m_node->loadVarargsData();
4398 LValue jsArguments = lowJSValue(m_node->child1());
4399
4400 LValue length = vmCall(
4401 m_out.operation(operationSizeOfVarargs), m_callFrame, jsArguments,
4402 m_out.constInt32(data->offset));
4403
4404 // FIXME: There is a chance that we will call an effectful length property twice. This is safe
4405 // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance
4406 // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right
4407 // past the sizing.
4408 // https://bugs.webkit.org/show_bug.cgi?id=141448
4409
4410 LValue lengthIncludingThis = m_out.add(length, m_out.int32One);
4411 speculate(
4412 VarargsOverflow, noValue(), nullptr,
4413 m_out.above(lengthIncludingThis, m_out.constInt32(data->limit)));
4414
4415 m_out.store32(lengthIncludingThis, payloadFor(data->machineCount));
4416
4417 // FIXME: This computation is rather silly. If operationLaodVarargs just took a pointer instead
4418 // of a VirtualRegister, we wouldn't have to do this.
4419 // https://bugs.webkit.org/show_bug.cgi?id=141660
4420 LValue machineStart = m_out.lShr(
4421 m_out.sub(addressFor(data->machineStart.offset()).value(), m_callFrame),
4422 m_out.constIntPtr(3));
4423
4424 vmCall(
4425 m_out.operation(operationLoadVarargs), m_callFrame,
4426 m_out.castToInt32(machineStart), jsArguments, m_out.constInt32(data->offset),
4427 length, m_out.constInt32(data->mandatoryMinimum));
4428 }
4429
4430 void compileForwardVarargs()
4431 {
4432 LoadVarargsData* data = m_node->loadVarargsData();
4433 InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame;
4434
4435 LValue length = getArgumentsLength(inlineCallFrame).value;
4436 LValue lengthIncludingThis = m_out.add(length, m_out.constInt32(1 - data->offset));
4437
4438 speculate(
4439 VarargsOverflow, noValue(), nullptr,
4440 m_out.above(lengthIncludingThis, m_out.constInt32(data->limit)));
4441
4442 m_out.store32(lengthIncludingThis, payloadFor(data->machineCount));
4443
4444 LValue sourceStart = getArgumentsStart(inlineCallFrame);
4445 LValue targetStart = addressFor(data->machineStart).value();
4446
4447 LBasicBlock undefinedLoop = FTL_NEW_BLOCK(m_out, ("ForwardVarargs undefined loop body"));
4448 LBasicBlock mainLoopEntry = FTL_NEW_BLOCK(m_out, ("ForwardVarargs main loop entry"));
4449 LBasicBlock mainLoop = FTL_NEW_BLOCK(m_out, ("ForwardVarargs main loop body"));
4450 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ForwardVarargs continuation"));
4451
4452 LValue lengthAsPtr = m_out.zeroExtPtr(length);
4453 ValueFromBlock loopBound = m_out.anchor(m_out.constIntPtr(data->mandatoryMinimum));
4454 m_out.branch(
4455 m_out.above(loopBound.value(), lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry));
4456
4457 LBasicBlock lastNext = m_out.appendTo(undefinedLoop, mainLoopEntry);
4458 LValue previousIndex = m_out.phi(m_out.intPtr, loopBound);
4459 LValue currentIndex = m_out.sub(previousIndex, m_out.intPtrOne);
4460 m_out.store64(
4461 m_out.constInt64(JSValue::encode(jsUndefined())),
4462 m_out.baseIndex(m_heaps.variables, targetStart, currentIndex));
4463 ValueFromBlock nextIndex = m_out.anchor(currentIndex);
4464 addIncoming(previousIndex, nextIndex);
4465 m_out.branch(
4466 m_out.above(currentIndex, lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry));
4467
4468 m_out.appendTo(mainLoopEntry, mainLoop);
4469 loopBound = m_out.anchor(lengthAsPtr);
4470 m_out.branch(m_out.notNull(loopBound.value()), unsure(mainLoop), unsure(continuation));
4471
4472 m_out.appendTo(mainLoop, continuation);
4473 previousIndex = m_out.phi(m_out.intPtr, loopBound);
4474 currentIndex = m_out.sub(previousIndex, m_out.intPtrOne);
4475 LValue value = m_out.load64(
4476 m_out.baseIndex(
4477 m_heaps.variables, sourceStart,
4478 m_out.add(currentIndex, m_out.constIntPtr(data->offset))));
4479 m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, currentIndex));
4480 nextIndex = m_out.anchor(currentIndex);
4481 addIncoming(previousIndex, nextIndex);
4482 m_out.branch(m_out.isNull(currentIndex), unsure(continuation), unsure(mainLoop));
4483
4484 m_out.appendTo(continuation, lastNext);
4485 }
4486
4487 void compileJump()
4488 {
4489 m_out.jump(lowBlock(m_node->targetBlock()));
4490 }
4491
4492 void compileBranch()
4493 {
4494 m_out.branch(
4495 boolify(m_node->child1()),
4496 WeightedTarget(
4497 lowBlock(m_node->branchData()->taken.block),
4498 m_node->branchData()->taken.count),
4499 WeightedTarget(
4500 lowBlock(m_node->branchData()->notTaken.block),
4501 m_node->branchData()->notTaken.count));
4502 }
4503
4504 void compileSwitch()
4505 {
4506 SwitchData* data = m_node->switchData();
4507 switch (data->kind) {
4508 case SwitchImm: {
4509 Vector<ValueFromBlock, 2> intValues;
4510 LBasicBlock switchOnInts = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm int case"));
4511
4512 LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts);
4513
4514 switch (m_node->child1().useKind()) {
4515 case Int32Use: {
4516 intValues.append(m_out.anchor(lowInt32(m_node->child1())));
4517 m_out.jump(switchOnInts);
4518 break;
4519 }
4520
4521 case UntypedUse: {
4522 LBasicBlock isInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is int"));
4523 LBasicBlock isNotInt = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is not int"));
4524 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("Switch/SwitchImm is double"));
4525
4526 LValue boxedValue = lowJSValue(m_node->child1());
4527 m_out.branch(isNotInt32(boxedValue), unsure(isNotInt), unsure(isInt));
4528
4529 LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt);
4530
4531 intValues.append(m_out.anchor(unboxInt32(boxedValue)));
4532 m_out.jump(switchOnInts);
4533
4534 m_out.appendTo(isNotInt, isDouble);
4535 m_out.branch(
4536 isCellOrMisc(boxedValue, provenType(m_node->child1())),
4537 usually(lowBlock(data->fallThrough.block)), rarely(isDouble));
4538
4539 m_out.appendTo(isDouble, innerLastNext);
4540 LValue doubleValue = unboxDouble(boxedValue);
4541 LValue intInDouble = m_out.fpToInt32(doubleValue);
4542 intValues.append(m_out.anchor(intInDouble));
4543 m_out.branch(
4544 m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue),
4545 unsure(switchOnInts), unsure(lowBlock(data->fallThrough.block)));
4546 break;
4547 }
4548
4549 default:
4550 DFG_CRASH(m_graph, m_node, "Bad use kind");
4551 break;
4552 }
4553
4554 m_out.appendTo(switchOnInts, lastNext);
4555 buildSwitch(data, m_out.int32, m_out.phi(m_out.int32, intValues));
4556 return;
4557 }
4558
4559 case SwitchChar: {
4560 LValue stringValue;
4561
4562 // FIXME: We should use something other than unsure() for the branch weight
4563 // of the fallThrough block. The main challenge is just that we have multiple
4564 // branches to fallThrough but a single count, so we would need to divvy it up
4565 // among the different lowered branches.
4566 // https://bugs.webkit.org/show_bug.cgi?id=129082
4567
4568 switch (m_node->child1().useKind()) {
4569 case StringUse: {
4570 stringValue = lowString(m_node->child1());
4571 break;
4572 }
4573
4574 case UntypedUse: {
4575 LValue unboxedValue = lowJSValue(m_node->child1());
4576
4577 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is cell"));
4578 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar is string"));
4579
4580 m_out.branch(
4581 isNotCell(unboxedValue, provenType(m_node->child1())),
4582 unsure(lowBlock(data->fallThrough.block)), unsure(isCellCase));
4583
4584 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
4585 LValue cellValue = unboxedValue;
4586 m_out.branch(
4587 isNotString(cellValue, provenType(m_node->child1())),
4588 unsure(lowBlock(data->fallThrough.block)), unsure(isStringCase));
4589
4590 m_out.appendTo(isStringCase, lastNext);
4591 stringValue = cellValue;
4592 break;
4593 }
4594
4595 default:
4596 DFG_CRASH(m_graph, m_node, "Bad use kind");
4597 break;
4598 }
4599
4600 LBasicBlock lengthIs1 = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar length is 1"));
4601 LBasicBlock needResolution = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolution"));
4602 LBasicBlock resolved = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar resolved"));
4603 LBasicBlock is8Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 8bit"));
4604 LBasicBlock is16Bit = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar 16bit"));
4605 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Switch/SwitchChar continuation"));
4606
4607 m_out.branch(
4608 m_out.notEqual(
4609 m_out.load32NonNegative(stringValue, m_heaps.JSString_length),
4610 m_out.int32One),
4611 unsure(lowBlock(data->fallThrough.block)), unsure(lengthIs1));
4612
4613 LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution);
4614 Vector<ValueFromBlock, 2> values;
4615 LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value);
4616 values.append(m_out.anchor(fastValue));
4617 m_out.branch(m_out.isNull(fastValue), rarely(needResolution), usually(resolved));
4618
4619 m_out.appendTo(needResolution, resolved);
4620 values.append(m_out.anchor(
4621 vmCall(m_out.operation(operationResolveRope), m_callFrame, stringValue)));
4622 m_out.jump(resolved);
4623
4624 m_out.appendTo(resolved, is8Bit);
4625 LValue value = m_out.phi(m_out.intPtr, values);
4626 LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data);
4627 m_out.branch(
4628 m_out.testNonZero32(
4629 m_out.load32(value, m_heaps.StringImpl_hashAndFlags),
4630 m_out.constInt32(StringImpl::flagIs8Bit())),
4631 unsure(is8Bit), unsure(is16Bit));
4632
4633 Vector<ValueFromBlock, 2> characters;
4634 m_out.appendTo(is8Bit, is16Bit);
4635 characters.append(m_out.anchor(
4636 m_out.zeroExt(m_out.load8(characterData, m_heaps.characters8[0]), m_out.int16)));
4637 m_out.jump(continuation);
4638
4639 m_out.appendTo(is16Bit, continuation);
4640 characters.append(m_out.anchor(m_out.load16(characterData, m_heaps.characters16[0])));
4641 m_out.jump(continuation);
4642
4643 m_out.appendTo(continuation, lastNext);
4644 buildSwitch(data, m_out.int16, m_out.phi(m_out.int16, characters));
4645 return;
4646 }
4647
4648 case SwitchString: {
4649 switch (m_node->child1().useKind()) {
4650 case StringIdentUse: {
4651 LValue stringImpl = lowStringIdent(m_node->child1());
4652
4653 Vector<SwitchCase> cases;
4654 for (unsigned i = 0; i < data->cases.size(); ++i) {
4655 LValue value = m_out.constIntPtr(data->cases[i].value.stringImpl());
4656 LBasicBlock block = lowBlock(data->cases[i].target.block);
4657 Weight weight = Weight(data->cases[i].target.count);
4658 cases.append(SwitchCase(value, block, weight));
4659 }
4660
4661 m_out.switchInstruction(
4662 stringImpl, cases, lowBlock(data->fallThrough.block),
4663 Weight(data->fallThrough.count));
4664 return;
4665 }
4666
4667 case StringUse: {
4668 switchString(data, lowString(m_node->child1()));
4669 return;
4670 }
4671
4672 case UntypedUse: {
4673 LValue value = lowJSValue(m_node->child1());
4674
4675 LBasicBlock isCellBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString Untyped cell case"));
4676 LBasicBlock isStringBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString Untyped string case"));
4677
4678 m_out.branch(
4679 isCell(value, provenType(m_node->child1())),
4680 unsure(isCellBlock), unsure(lowBlock(data->fallThrough.block)));
4681
4682 LBasicBlock lastNext = m_out.appendTo(isCellBlock, isStringBlock);
4683
4684 m_out.branch(
4685 isString(value, provenType(m_node->child1())),
4686 unsure(isStringBlock), unsure(lowBlock(data->fallThrough.block)));
4687
4688 m_out.appendTo(isStringBlock, lastNext);
4689
4690 switchString(data, value);
4691 return;
4692 }
4693
4694 default:
4695 DFG_CRASH(m_graph, m_node, "Bad use kind");
4696 return;
4697 }
4698 return;
4699 }
4700
4701 case SwitchCell: {
4702 LValue cell;
4703 switch (m_node->child1().useKind()) {
4704 case CellUse: {
4705 cell = lowCell(m_node->child1());
4706 break;
4707 }
4708
4709 case UntypedUse: {
4710 LValue value = lowJSValue(m_node->child1());
4711 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("Switch/SwitchCell cell case"));
4712 m_out.branch(
4713 isCell(value, provenType(m_node->child1())),
4714 unsure(cellCase), unsure(lowBlock(data->fallThrough.block)));
4715 m_out.appendTo(cellCase);
4716 cell = value;
4717 break;
4718 }
4719
4720 default:
4721 DFG_CRASH(m_graph, m_node, "Bad use kind");
4722 return;
4723 }
4724
4725 buildSwitch(m_node->switchData(), m_out.intPtr, cell);
4726 return;
4727 } }
4728
4729 DFG_CRASH(m_graph, m_node, "Bad switch kind");
4730 }
4731
4732 void compileReturn()
4733 {
4734 m_out.ret(lowJSValue(m_node->child1()));
4735 }
4736
4737 void compileForceOSRExit()
4738 {
4739 terminate(InadequateCoverage);
4740 }
4741
4742 void compileThrow()
4743 {
4744 terminate(Uncountable);
4745 }
4746
4747 void compileInvalidationPoint()
4748 {
4749 if (verboseCompilationEnabled())
4750 dataLog(" Invalidation point with availability: ", availabilityMap(), "\n");
4751
4752 m_ftlState.jitCode->osrExit.append(OSRExit(
4753 UncountableInvalidation, InvalidValueFormat, MethodOfGettingAValueProfile(),
4754 m_codeOriginForExitTarget, m_codeOriginForExitProfile,
4755 availabilityMap().m_locals.numberOfArguments(),
4756 availabilityMap().m_locals.numberOfLocals()));
4757 m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo());
4758
4759 OSRExit& exit = m_ftlState.jitCode->osrExit.last();
4760 OSRExitCompilationInfo& info = m_ftlState.finalizer->osrExit.last();
4761
4762 ExitArgumentList arguments;
4763
4764 buildExitArguments(exit, arguments, FormattedValue(), exit.m_codeOrigin);
4765 callStackmap(exit, arguments);
4766
4767 info.m_isInvalidationPoint = true;
4768 }
4769
4770 void compileIsUndefined()
4771 {
4772 setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined));
4773 }
4774
4775 void compileIsBoolean()
4776 {
4777 setBoolean(isBoolean(lowJSValue(m_node->child1()), provenType(m_node->child1())));
4778 }
4779
4780 void compileIsNumber()
4781 {
4782 setBoolean(isNumber(lowJSValue(m_node->child1()), provenType(m_node->child1())));
4783 }
4784
4785 void compileIsString()
4786 {
4787 LValue value = lowJSValue(m_node->child1());
4788
4789 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("IsString cell case"));
4790 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsString continuation"));
4791
4792 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4793 m_out.branch(
4794 isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
4795
4796 LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
4797 ValueFromBlock cellResult = m_out.anchor(isString(value, provenType(m_node->child1())));
4798 m_out.jump(continuation);
4799
4800 m_out.appendTo(continuation, lastNext);
4801 setBoolean(m_out.phi(m_out.boolean, notCellResult, cellResult));
4802 }
4803
4804 void compileIsObject()
4805 {
4806 LValue value = lowJSValue(m_node->child1());
4807
4808 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("IsObject cell case"));
4809 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsObject continuation"));
4810
4811 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4812 m_out.branch(
4813 isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation));
4814
4815 LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation);
4816 ValueFromBlock cellResult = m_out.anchor(isObject(value, provenType(m_node->child1())));
4817 m_out.jump(continuation);
4818
4819 m_out.appendTo(continuation, lastNext);
4820 setBoolean(m_out.phi(m_out.boolean, notCellResult, cellResult));
4821 }
4822
4823 void compileIsObjectOrNull()
4824 {
4825 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
4826
4827 Edge child = m_node->child1();
4828 LValue value = lowJSValue(child);
4829
4830 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull cell case"));
4831 LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull not function case"));
4832 LBasicBlock objectCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull object case"));
4833 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull slow path"));
4834 LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull not cell case"));
4835 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsObjectOrNull continuation"));
4836
4837 m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase));
4838
4839 LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase);
4840 ValueFromBlock isFunctionResult = m_out.anchor(m_out.booleanFalse);
4841 m_out.branch(
4842 isFunction(value, provenType(child)),
4843 unsure(continuation), unsure(notFunctionCase));
4844
4845 m_out.appendTo(notFunctionCase, objectCase);
4846 ValueFromBlock notObjectResult = m_out.anchor(m_out.booleanFalse);
4847 m_out.branch(
4848 isObject(value, provenType(child)),
4849 unsure(objectCase), unsure(continuation));
4850
4851 m_out.appendTo(objectCase, slowPath);
4852 ValueFromBlock objectResult = m_out.anchor(m_out.booleanTrue);
4853 m_out.branch(
4854 isExoticForTypeof(value, provenType(child)),
4855 rarely(slowPath), usually(continuation));
4856
4857 m_out.appendTo(slowPath, notCellCase);
4858 LValue slowResultValue = vmCall(
4859 m_out.operation(operationObjectIsObject), m_callFrame, weakPointer(globalObject),
4860 value);
4861 ValueFromBlock slowResult = m_out.anchor(m_out.notNull(slowResultValue));
4862 m_out.jump(continuation);
4863
4864 m_out.appendTo(notCellCase, continuation);
4865 LValue notCellResultValue = m_out.equal(value, m_out.constInt64(JSValue::encode(jsNull())));
4866 ValueFromBlock notCellResult = m_out.anchor(notCellResultValue);
4867 m_out.jump(continuation);
4868
4869 m_out.appendTo(continuation, lastNext);
4870 LValue result = m_out.phi(
4871 m_out.boolean,
4872 isFunctionResult, notObjectResult, objectResult, slowResult, notCellResult);
4873 setBoolean(result);
4874 }
4875
4876 void compileIsFunction()
4877 {
4878 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
4879
4880 Edge child = m_node->child1();
4881 LValue value = lowJSValue(child);
4882
4883 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("IsFunction cell case"));
4884 LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("IsFunction not function case"));
4885 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("IsFunction slow path"));
4886 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("IsFunction continuation"));
4887
4888 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4889 m_out.branch(
4890 isCell(value, provenType(child)), unsure(cellCase), unsure(continuation));
4891
4892 LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase);
4893 ValueFromBlock functionResult = m_out.anchor(m_out.booleanTrue);
4894 m_out.branch(
4895 isFunction(value, provenType(child)),
4896 unsure(continuation), unsure(notFunctionCase));
4897
4898 m_out.appendTo(notFunctionCase, slowPath);
4899 ValueFromBlock objectResult = m_out.anchor(m_out.booleanFalse);
4900 m_out.branch(
4901 isExoticForTypeof(value, provenType(child)),
4902 rarely(slowPath), usually(continuation));
4903
4904 m_out.appendTo(slowPath, continuation);
4905 LValue slowResultValue = vmCall(
4906 m_out.operation(operationObjectIsFunction), m_callFrame, weakPointer(globalObject),
4907 value);
4908 ValueFromBlock slowResult = m_out.anchor(m_out.notNull(slowResultValue));
4909 m_out.jump(continuation);
4910
4911 m_out.appendTo(continuation, lastNext);
4912 LValue result = m_out.phi(
4913 m_out.boolean, notCellResult, functionResult, objectResult, slowResult);
4914 setBoolean(result);
4915 }
4916
4917 void compileTypeOf()
4918 {
4919 Edge child = m_node->child1();
4920 LValue value = lowJSValue(child);
4921
4922 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("TypeOf continuation"));
4923 LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
4924
4925 Vector<ValueFromBlock> results;
4926
4927 buildTypeOf(
4928 child, value,
4929 [&] (TypeofType type) {
4930 results.append(m_out.anchor(weakPointer(vm().smallStrings.typeString(type))));
4931 m_out.jump(continuation);
4932 });
4933
4934 m_out.appendTo(continuation, lastNext);
4935 setJSValue(m_out.phi(m_out.int64, results));
4936 }
4937
4938 void compileIn()
4939 {
4940 Edge base = m_node->child2();
4941 LValue cell = lowCell(base);
4942 speculateObject(base, cell);
4943 if (JSString* string = m_node->child1()->dynamicCastConstant<JSString*>()) {
4944 if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) {
4945
4946 const auto str = static_cast<const AtomicStringImpl*>(string->tryGetValueImpl());
4947 unsigned stackmapID = m_stackmapIDs++;
4948
4949 LValue call = m_out.call(
4950 m_out.patchpointInt64Intrinsic(),
4951 m_out.constInt64(stackmapID), m_out.constInt32(sizeOfIn()),
4952 constNull(m_out.ref8), m_out.constInt32(1), cell);
4953
4954 setInstructionCallingConvention(call, LLVMAnyRegCallConv);
4955
4956 m_ftlState.checkIns.append(CheckInDescriptor(stackmapID, m_node->origin.semantic, str));
4957 setJSValue(call);
4958 return;
4959 }
4960 }
4961
4962 setJSValue(vmCall(m_out.operation(operationGenericIn), m_callFrame, cell, lowJSValue(m_node->child1())));
4963 }
4964
4965 void compileCheckHasInstance()
4966 {
4967 speculate(
4968 Uncountable, noValue(), 0,
4969 m_out.testIsZero8(
4970 m_out.load8(lowCell(m_node->child1()), m_heaps.JSCell_typeInfoFlags),
4971 m_out.constInt8(ImplementsDefaultHasInstance)));
4972 }
4973
4974 void compileInstanceOf()
4975 {
4976 LValue cell;
4977
4978 if (m_node->child1().useKind() == UntypedUse)
4979 cell = lowJSValue(m_node->child1());
4980 else
4981 cell = lowCell(m_node->child1());
4982
4983 LValue prototype = lowCell(m_node->child2());
4984
4985 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("InstanceOf cell case"));
4986 LBasicBlock loop = FTL_NEW_BLOCK(m_out, ("InstanceOf loop"));
4987 LBasicBlock notYetInstance = FTL_NEW_BLOCK(m_out, ("InstanceOf not yet instance"));
4988 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("InstanceOf continuation"));
4989
4990 LValue condition;
4991 if (m_node->child1().useKind() == UntypedUse)
4992 condition = isCell(cell, provenType(m_node->child1()));
4993 else
4994 condition = m_out.booleanTrue;
4995
4996 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
4997 m_out.branch(condition, unsure(isCellCase), unsure(continuation));
4998
4999 LBasicBlock lastNext = m_out.appendTo(isCellCase, loop);
5000
5001 speculate(BadType, noValue(), 0, isNotObject(prototype, provenType(m_node->child2())));
5002
5003 ValueFromBlock originalValue = m_out.anchor(cell);
5004 m_out.jump(loop);
5005
5006 m_out.appendTo(loop, notYetInstance);
5007 LValue value = m_out.phi(m_out.int64, originalValue);
5008 LValue structure = loadStructure(value);
5009 LValue currentPrototype = m_out.load64(structure, m_heaps.Structure_prototype);
5010 ValueFromBlock isInstanceResult = m_out.anchor(m_out.booleanTrue);
5011 m_out.branch(
5012 m_out.equal(currentPrototype, prototype),
5013 unsure(continuation), unsure(notYetInstance));
5014
5015 m_out.appendTo(notYetInstance, continuation);
5016 ValueFromBlock notInstanceResult = m_out.anchor(m_out.booleanFalse);
5017 addIncoming(value, m_out.anchor(currentPrototype));
5018 m_out.branch(isCell(currentPrototype), unsure(loop), unsure(continuation));
5019
5020 m_out.appendTo(continuation, lastNext);
5021 setBoolean(
5022 m_out.phi(m_out.boolean, notCellResult, isInstanceResult, notInstanceResult));
5023 }
5024
5025 void compileCountExecution()
5026 {
5027 TypedPointer counter = m_out.absolute(m_node->executionCounter()->address());
5028 m_out.store64(m_out.add(m_out.load64(counter), m_out.constInt64(1)), counter);
5029 }
5030
5031 void compileStoreBarrier()
5032 {
5033 emitStoreBarrier(lowCell(m_node->child1()));
5034 }
5035
5036 void compileHasIndexedProperty()
5037 {
5038 switch (m_node->arrayMode().type()) {
5039 case Array::Int32:
5040 case Array::Contiguous: {
5041 LValue base = lowCell(m_node->child1());
5042 LValue index = lowInt32(m_node->child2());
5043 LValue storage = lowStorage(m_node->child3());
5044
5045 IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ?
5046 m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties;
5047
5048 LBasicBlock checkHole = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous check hole"));
5049 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous slow case"));
5050 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty int/contiguous continuation"));
5051
5052 if (!m_node->arrayMode().isInBounds()) {
5053 m_out.branch(
5054 m_out.aboveOrEqual(
5055 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
5056 rarely(slowCase), usually(checkHole));
5057 } else
5058 m_out.jump(checkHole);
5059
5060 LBasicBlock lastNext = m_out.appendTo(checkHole, slowCase);
5061 ValueFromBlock checkHoleResult = m_out.anchor(
5062 m_out.notZero64(m_out.load64(baseIndex(heap, storage, index, m_node->child2()))));
5063 m_out.branch(checkHoleResult.value(), usually(continuation), rarely(slowCase));
5064
5065 m_out.appendTo(slowCase, continuation);
5066 ValueFromBlock slowResult = m_out.anchor(m_out.equal(
5067 m_out.constInt64(JSValue::encode(jsBoolean(true))),
5068 vmCall(m_out.operation(operationHasIndexedProperty), m_callFrame, base, index)));
5069 m_out.jump(continuation);
5070
5071 m_out.appendTo(continuation, lastNext);
5072 setBoolean(m_out.phi(m_out.boolean, checkHoleResult, slowResult));
5073 return;
5074 }
5075 case Array::Double: {
5076 LValue base = lowCell(m_node->child1());
5077 LValue index = lowInt32(m_node->child2());
5078 LValue storage = lowStorage(m_node->child3());
5079
5080 IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties;
5081
5082 LBasicBlock checkHole = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double check hole"));
5083 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double slow case"));
5084 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasIndexedProperty double continuation"));
5085
5086 if (!m_node->arrayMode().isInBounds()) {
5087 m_out.branch(
5088 m_out.aboveOrEqual(
5089 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)),
5090 rarely(slowCase), usually(checkHole));
5091 } else
5092 m_out.jump(checkHole);
5093
5094 LBasicBlock lastNext = m_out.appendTo(checkHole, slowCase);
5095 LValue doubleValue = m_out.loadDouble(baseIndex(heap, storage, index, m_node->child2()));
5096 ValueFromBlock checkHoleResult = m_out.anchor(m_out.doubleEqual(doubleValue, doubleValue));
5097 m_out.branch(checkHoleResult.value(), usually(continuation), rarely(slowCase));
5098
5099 m_out.appendTo(slowCase, continuation);
5100 ValueFromBlock slowResult = m_out.anchor(m_out.equal(
5101 m_out.constInt64(JSValue::encode(jsBoolean(true))),
5102 vmCall(m_out.operation(operationHasIndexedProperty), m_callFrame, base, index)));
5103 m_out.jump(continuation);
5104
5105 m_out.appendTo(continuation, lastNext);
5106 setBoolean(m_out.phi(m_out.boolean, checkHoleResult, slowResult));
5107 return;
5108 }
5109
5110 default:
5111 RELEASE_ASSERT_NOT_REACHED();
5112 return;
5113 }
5114 }
5115
5116 void compileHasGenericProperty()
5117 {
5118 LValue base = lowJSValue(m_node->child1());
5119 LValue property = lowCell(m_node->child2());
5120 setJSValue(vmCall(m_out.operation(operationHasGenericProperty), m_callFrame, base, property));
5121 }
5122
5123 void compileHasStructureProperty()
5124 {
5125 LValue base = lowJSValue(m_node->child1());
5126 LValue property = lowString(m_node->child2());
5127 LValue enumerator = lowCell(m_node->child3());
5128
5129 LBasicBlock correctStructure = FTL_NEW_BLOCK(m_out, ("HasStructureProperty correct structure"));
5130 LBasicBlock wrongStructure = FTL_NEW_BLOCK(m_out, ("HasStructureProperty wrong structure"));
5131 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("HasStructureProperty continuation"));
5132
5133 m_out.branch(m_out.notEqual(
5134 m_out.load32(base, m_heaps.JSCell_structureID),
5135 m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)),
5136 rarely(wrongStructure), usually(correctStructure));
5137
5138 LBasicBlock lastNext = m_out.appendTo(correctStructure, wrongStructure);
5139 ValueFromBlock correctStructureResult = m_out.anchor(m_out.booleanTrue);
5140 m_out.jump(continuation);
5141
5142 m_out.appendTo(wrongStructure, continuation);
5143 ValueFromBlock wrongStructureResult = m_out.anchor(
5144 m_out.equal(
5145 m_out.constInt64(JSValue::encode(jsBoolean(true))),
5146 vmCall(m_out.operation(operationHasGenericProperty), m_callFrame, base, property)));
5147 m_out.jump(continuation);
5148
5149 m_out.appendTo(continuation, lastNext);
5150 setBoolean(m_out.phi(m_out.boolean, correctStructureResult, wrongStructureResult));
5151 }
5152
5153 void compileGetDirectPname()
5154 {
5155 LValue base = lowCell(m_graph.varArgChild(m_node, 0));
5156 LValue property = lowCell(m_graph.varArgChild(m_node, 1));
5157 LValue index = lowInt32(m_graph.varArgChild(m_node, 2));
5158 LValue enumerator = lowCell(m_graph.varArgChild(m_node, 3));
5159
5160 LBasicBlock checkOffset = FTL_NEW_BLOCK(m_out, ("GetDirectPname check offset"));
5161 LBasicBlock inlineLoad = FTL_NEW_BLOCK(m_out, ("GetDirectPname inline load"));
5162 LBasicBlock outOfLineLoad = FTL_NEW_BLOCK(m_out, ("GetDirectPname out-of-line load"));
5163 LBasicBlock slowCase = FTL_NEW_BLOCK(m_out, ("GetDirectPname slow case"));
5164 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetDirectPname continuation"));
5165
5166 m_out.branch(m_out.notEqual(
5167 m_out.load32(base, m_heaps.JSCell_structureID),
5168 m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)),
5169 rarely(slowCase), usually(checkOffset));
5170
5171 LBasicBlock lastNext = m_out.appendTo(checkOffset, inlineLoad);
5172 m_out.branch(m_out.aboveOrEqual(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity)),
5173 unsure(outOfLineLoad), unsure(inlineLoad));
5174
5175 m_out.appendTo(inlineLoad, outOfLineLoad);
5176 ValueFromBlock inlineResult = m_out.anchor(
5177 m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(),
5178 base, m_out.zeroExt(index, m_out.int64), ScaleEight, JSObject::offsetOfInlineStorage())));
5179 m_out.jump(continuation);
5180
5181 m_out.appendTo(outOfLineLoad, slowCase);
5182 LValue storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly);
5183 LValue realIndex = m_out.signExt(
5184 m_out.neg(m_out.sub(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity))),
5185 m_out.int64);
5186 int32_t offsetOfFirstProperty = static_cast<int32_t>(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue);
5187 ValueFromBlock outOfLineResult = m_out.anchor(
5188 m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), storage, realIndex, ScaleEight, offsetOfFirstProperty)));
5189 m_out.jump(continuation);
5190
5191 m_out.appendTo(slowCase, continuation);
5192 ValueFromBlock slowCaseResult = m_out.anchor(
5193 vmCall(m_out.operation(operationGetByVal), m_callFrame, base, property));
5194 m_out.jump(continuation);
5195
5196 m_out.appendTo(continuation, lastNext);
5197 setJSValue(m_out.phi(m_out.int64, inlineResult, outOfLineResult, slowCaseResult));
5198 }
5199
5200 void compileGetEnumerableLength()
5201 {
5202 LValue enumerator = lowCell(m_node->child1());
5203 setInt32(m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_indexLength));
5204 }
5205
5206 void compileGetPropertyEnumerator()
5207 {
5208 LValue base = lowCell(m_node->child1());
5209 setJSValue(vmCall(m_out.operation(operationGetPropertyEnumerator), m_callFrame, base));
5210 }
5211
5212 void compileGetEnumeratorStructurePname()
5213 {
5214 LValue enumerator = lowCell(m_node->child1());
5215 LValue index = lowInt32(m_node->child2());
5216
5217 LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname in bounds"));
5218 LBasicBlock outOfBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname out of bounds"));
5219 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetEnumeratorStructurePname continuation"));
5220
5221 m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endStructurePropertyIndex)),
5222 usually(inBounds), rarely(outOfBounds));
5223
5224 LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds);
5225 LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector);
5226 ValueFromBlock inBoundsResult = m_out.anchor(
5227 m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index))));
5228 m_out.jump(continuation);
5229
5230 m_out.appendTo(outOfBounds, continuation);
5231 ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueNull));
5232 m_out.jump(continuation);
5233
5234 m_out.appendTo(continuation, lastNext);
5235 setJSValue(m_out.phi(m_out.int64, inBoundsResult, outOfBoundsResult));
5236 }
5237
5238 void compileGetEnumeratorGenericPname()
5239 {
5240 LValue enumerator = lowCell(m_node->child1());
5241 LValue index = lowInt32(m_node->child2());
5242
5243 LBasicBlock inBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname in bounds"));
5244 LBasicBlock outOfBounds = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname out of bounds"));
5245 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("GetEnumeratorGenericPname continuation"));
5246
5247 m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endGenericPropertyIndex)),
5248 usually(inBounds), rarely(outOfBounds));
5249
5250 LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds);
5251 LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector);
5252 ValueFromBlock inBoundsResult = m_out.anchor(
5253 m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index))));
5254 m_out.jump(continuation);
5255
5256 m_out.appendTo(outOfBounds, continuation);
5257 ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueNull));
5258 m_out.jump(continuation);
5259
5260 m_out.appendTo(continuation, lastNext);
5261 setJSValue(m_out.phi(m_out.int64, inBoundsResult, outOfBoundsResult));
5262 }
5263
5264 void compileToIndexString()
5265 {
5266 LValue index = lowInt32(m_node->child1());
5267 setJSValue(vmCall(m_out.operation(operationToIndexString), m_callFrame, index));
5268 }
5269
5270 void compileCheckStructureImmediate()
5271 {
5272 LValue structure = lowCell(m_node->child1());
5273 checkStructure(
5274 structure, noValue(), BadCache, m_node->structureSet(),
5275 [this] (Structure* structure) {
5276 return weakStructure(structure);
5277 });
5278 }
5279
5280 void compileMaterializeNewObject()
5281 {
5282 ObjectMaterializationData& data = m_node->objectMaterializationData();
5283
5284 // Lower the values first, to avoid creating values inside a control flow diamond.
5285
5286 Vector<LValue, 8> values;
5287 for (unsigned i = 0; i < data.m_properties.size(); ++i)
5288 values.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
5289
5290 StructureSet set;
5291 m_interpreter.phiChildren()->forAllTransitiveIncomingValues(
5292 m_graph.varArgChild(m_node, 0).node(),
5293 [&] (Node* incoming) {
5294 set.add(incoming->castConstant<Structure*>());
5295 });
5296
5297 Vector<LBasicBlock, 1> blocks(set.size());
5298 for (unsigned i = set.size(); i--;)
5299 blocks[i] = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject case ", i));
5300 LBasicBlock dummyDefault = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject default case"));
5301 LBasicBlock outerContinuation = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject continuation"));
5302
5303 Vector<SwitchCase, 1> cases(set.size());
5304 for (unsigned i = set.size(); i--;)
5305 cases[i] = SwitchCase(weakStructure(set[i]), blocks[i], Weight(1));
5306 m_out.switchInstruction(
5307 lowCell(m_graph.varArgChild(m_node, 0)), cases, dummyDefault, Weight(0));
5308
5309 LBasicBlock outerLastNext = m_out.m_nextBlock;
5310
5311 Vector<ValueFromBlock, 1> results;
5312
5313 for (unsigned i = set.size(); i--;) {
5314 m_out.appendTo(blocks[i], i + 1 < set.size() ? blocks[i + 1] : dummyDefault);
5315
5316 Structure* structure = set[i];
5317
5318 LValue object;
5319 LValue butterfly;
5320
5321 if (structure->outOfLineCapacity()) {
5322 size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
5323 MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize);
5324
5325 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject complex object allocation slow path"));
5326 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MaterializeNewObject complex object allocation continuation"));
5327
5328 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5329
5330 LValue endOfStorage = allocateBasicStorageAndGetEnd(
5331 m_out.constIntPtr(structure->outOfLineCapacity() * sizeof(JSValue)),
5332 slowPath);
5333
5334 LValue fastButterflyValue = m_out.add(
5335 m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage);
5336
5337 LValue fastObjectValue = allocateObject(
5338 m_out.constIntPtr(allocator), structure, fastButterflyValue, slowPath);
5339
5340 ValueFromBlock fastObject = m_out.anchor(fastObjectValue);
5341 ValueFromBlock fastButterfly = m_out.anchor(fastButterflyValue);
5342 m_out.jump(continuation);
5343
5344 m_out.appendTo(slowPath, continuation);
5345
5346 ValueFromBlock slowObject = m_out.anchor(vmCall(
5347 m_out.operation(operationNewObjectWithButterfly),
5348 m_callFrame, m_out.constIntPtr(structure)));
5349 ValueFromBlock slowButterfly = m_out.anchor(
5350 m_out.loadPtr(slowObject.value(), m_heaps.JSObject_butterfly));
5351
5352 m_out.jump(continuation);
5353
5354 m_out.appendTo(continuation, lastNext);
5355
5356 object = m_out.phi(m_out.intPtr, fastObject, slowObject);
5357 butterfly = m_out.phi(m_out.intPtr, fastButterfly, slowButterfly);
5358 } else {
5359 // In the easy case where we can do a one-shot allocation, we simply allocate the
5360 // object to directly have the desired structure.
5361 object = allocateObject(structure);
5362 butterfly = nullptr; // Don't have one, don't need one.
5363 }
5364
5365 for (PropertyMapEntry entry : structure->getPropertiesConcurrently()) {
5366 for (unsigned i = data.m_properties.size(); i--;) {
5367 PhantomPropertyValue value = data.m_properties[i];
5368 if (m_graph.identifiers()[value.m_identifierNumber] != entry.key)
5369 continue;
5370
5371 LValue base = isInlineOffset(entry.offset) ? object : butterfly;
5372 storeProperty(values[i], base, value.m_identifierNumber, entry.offset);
5373 break;
5374 }
5375 }
5376
5377 results.append(m_out.anchor(object));
5378 m_out.jump(outerContinuation);
5379 }
5380
5381 m_out.appendTo(dummyDefault, outerContinuation);
5382 m_out.unreachable();
5383
5384 m_out.appendTo(outerContinuation, outerLastNext);
5385 setJSValue(m_out.phi(m_out.intPtr, results));
5386 }
5387
5388 void compileMaterializeCreateActivation()
5389 {
5390 ObjectMaterializationData& data = m_node->objectMaterializationData();
5391
5392 Vector<LValue, 8> values;
5393 for (unsigned i = 0; i < data.m_properties.size(); ++i)
5394 values.append(lowJSValue(m_graph.varArgChild(m_node, 1 + i)));
5395
5396 LValue scope = lowCell(m_graph.varArgChild(m_node, 0));
5397 SymbolTable* table = m_node->castOperand<SymbolTable*>();
5398 Structure* structure = m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure();
5399
5400 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("MaterializeCreateActivation slow path"));
5401 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("MaterializeCreateActivation continuation"));
5402
5403 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5404
5405 LValue fastObject = allocateObject<JSLexicalEnvironment>(
5406 JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath);
5407
5408 m_out.storePtr(scope, fastObject, m_heaps.JSScope_next);
5409 m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable);
5410
5411
5412 ValueFromBlock fastResult = m_out.anchor(fastObject);
5413 m_out.jump(continuation);
5414
5415 m_out.appendTo(slowPath, continuation);
5416 LValue callResult = vmCall(
5417 m_out.operation(operationCreateActivationDirect), m_callFrame, weakPointer(structure),
5418 scope, weakPointer(table));
5419 ValueFromBlock slowResult = m_out.anchor(callResult);
5420 m_out.jump(continuation);
5421
5422 m_out.appendTo(continuation, lastNext);
5423 LValue activation = m_out.phi(m_out.intPtr, fastResult, slowResult);
5424 RELEASE_ASSERT(data.m_properties.size() == table->scopeSize());
5425 for (unsigned i = 0; i < data.m_properties.size(); ++i) {
5426 m_out.store64(values[i],
5427 activation,
5428 m_heaps.JSEnvironmentRecord_variables[data.m_properties[i].m_identifierNumber]);
5429 }
5430
5431 if (validationEnabled()) {
5432 // Validate to make sure every slot in the scope has one value.
5433 ConcurrentJITLocker locker(table->m_lock);
5434 for (auto iter = table->begin(locker), end = table->end(locker); iter != end; ++iter) {
5435 bool found = false;
5436 for (unsigned i = 0; i < data.m_properties.size(); ++i) {
5437 if (iter->value.scopeOffset().offset() == data.m_properties[i].m_identifierNumber) {
5438 found = true;
5439 break;
5440 }
5441 }
5442 ASSERT_UNUSED(found, found);
5443 }
5444 }
5445
5446 setJSValue(activation);
5447 }
5448
5449#if ENABLE(FTL_NATIVE_CALL_INLINING)
5450 LValue getFunctionBySymbol(const CString symbol)
5451 {
5452 if (!m_ftlState.symbolTable.contains(symbol))
5453 return nullptr;
5454 if (!getModuleByPathForSymbol(m_ftlState.symbolTable.get(symbol), symbol))
5455 return nullptr;
5456 return getNamedFunction(m_ftlState.module, symbol.data());
5457 }
5458
5459 bool getModuleByPathForSymbol(const CString path, const CString symbol)
5460 {
5461 if (m_ftlState.nativeLoadedLibraries.contains(path)) {
5462 LValue function = getNamedFunction(m_ftlState.module, symbol.data());
5463 if (!isInlinableSize(function)) {
5464 // We had no choice but to compile this function, but don't try to inline it ever again.
5465 m_ftlState.symbolTable.remove(symbol);
5466 return false;
5467 }
5468 return true;
5469 }
5470
5471 LMemoryBuffer memBuf;
5472
5473 ASSERT(isX86() || isARM64());
5474
5475#if PLATFORM(EFL)
5476 const CString actualPath = toCString(bundlePath().data(), "/runtime/", path.data());
5477#else
5478 const CString actualPath = toCString(bundlePath().data(),
5479 isX86() ? "/Resources/Runtime/x86_64/" : "/Resources/Runtime/arm64/",
5480 path.data());
5481#endif
5482
5483 char* outMsg;
5484
5485 if (createMemoryBufferWithContentsOfFile(actualPath.data(), &memBuf, &outMsg)) {
5486 if (Options::verboseFTLFailure())
5487 dataLog("Failed to load module at ", actualPath, "\n for symbol ", symbol, "\nERROR: ", outMsg, "\n");
5488 disposeMessage(outMsg);
5489 return false;
5490 }
5491
5492 LModule module;
5493
5494 if (parseBitcodeInContext(m_ftlState.context, memBuf, &module, &outMsg)) {
5495 if (Options::verboseFTLFailure())
5496 dataLog("Failed to parse module at ", actualPath, "\n for symbol ", symbol, "\nERROR: ", outMsg, "\n");
5497 disposeMemoryBuffer(memBuf);
5498 disposeMessage(outMsg);
5499 return false;
5500 }
5501
5502 disposeMemoryBuffer(memBuf);
5503
5504 if (LValue function = getNamedFunction(m_ftlState.module, symbol.data())) {
5505 if (!isInlinableSize(function)) {
5506 m_ftlState.symbolTable.remove(symbol);
5507 disposeModule(module);
5508 return false;
5509 }
5510 }
5511
5512 Vector<CString> namedFunctions;
5513 for (LValue function = getFirstFunction(module); function; function = getNextFunction(function)) {
5514 CString functionName(getValueName(function));
5515 namedFunctions.append(functionName);
5516
5517 for (LBasicBlock basicBlock = getFirstBasicBlock(function); basicBlock; basicBlock = getNextBasicBlock(basicBlock)) {
5518 for (LValue instruction = getFirstInstruction(basicBlock); instruction; instruction = getNextInstruction(instruction)) {
5519 setMetadata(instruction, m_tbaaKind, nullptr);
5520 setMetadata(instruction, m_tbaaStructKind, nullptr);
5521 }
5522 }
5523 }
5524
5525 Vector<CString> namedGlobals;
5526 for (LValue global = getFirstGlobal(module); global; global = getNextGlobal(global)) {
5527 CString globalName(getValueName(global));
5528 namedGlobals.append(globalName);
5529 }
5530
5531 if (linkModules(m_ftlState.module, module, LLVMLinkerDestroySource, &outMsg)) {
5532 if (Options::verboseFTLFailure())
5533 dataLog("Failed to link module at ", actualPath, "\n for symbol ", symbol, "\nERROR: ", outMsg, "\n");
5534 disposeMessage(outMsg);
5535 return false;
5536 }
5537
5538 for (CString* symbol = namedFunctions.begin(); symbol != namedFunctions.end(); ++symbol) {
5539 LValue function = getNamedFunction(m_ftlState.module, symbol->data());
5540 LLVMLinkage linkage = getLinkage(function);
5541 if (linkage != LLVMInternalLinkage && linkage != LLVMPrivateLinkage)
5542 setVisibility(function, LLVMHiddenVisibility);
5543 if (!isDeclaration(function)) {
5544 setLinkage(function, LLVMPrivateLinkage);
5545 setLinkage(function, LLVMAvailableExternallyLinkage);
5546
5547 if (ASSERT_DISABLED)
5548 removeFunctionAttr(function, LLVMStackProtectAttribute);
5549 }
5550 }
5551
5552 for (CString* symbol = namedGlobals.begin(); symbol != namedGlobals.end(); ++symbol) {
5553 LValue global = getNamedGlobal(m_ftlState.module, symbol->data());
5554 LLVMLinkage linkage = getLinkage(global);
5555 if (linkage != LLVMInternalLinkage && linkage != LLVMPrivateLinkage)
5556 setVisibility(global, LLVMHiddenVisibility);
5557 if (!isDeclaration(global))
5558 setLinkage(global, LLVMPrivateLinkage);
5559 }
5560
5561 m_ftlState.nativeLoadedLibraries.add(path);
5562 return true;
5563 }
5564#endif
5565
5566 bool isInlinableSize(LValue function)
5567 {
5568 size_t instructionCount = 0;
5569 size_t maxSize = Options::maximumLLVMInstructionCountForNativeInlining();
5570 for (LBasicBlock basicBlock = getFirstBasicBlock(function); basicBlock; basicBlock = getNextBasicBlock(basicBlock)) {
5571 for (LValue instruction = getFirstInstruction(basicBlock); instruction; instruction = getNextInstruction(instruction)) {
5572 if (++instructionCount >= maxSize)
5573 return false;
5574 }
5575 }
5576 return true;
5577 }
5578
5579 LValue didOverflowStack()
5580 {
5581 // This does a very simple leaf function analysis. The invariant of FTL call
5582 // frames is that the caller had already done enough of a stack check to
5583 // prove that this call frame has enough stack to run, and also enough stack
5584 // to make runtime calls. So, we only need to stack check when making calls
5585 // to other JS functions. If we don't find such calls then we don't need to
5586 // do any stack checks.
5587
5588 for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
5589 BasicBlock* block = m_graph.block(blockIndex);
5590 if (!block)
5591 continue;
5592
5593 for (unsigned nodeIndex = block->size(); nodeIndex--;) {
5594 Node* node = block->at(nodeIndex);
5595
5596 switch (node->op()) {
5597 case GetById:
5598 case PutById:
5599 case Call:
5600 case Construct:
5601 case NativeCall:
5602 case NativeConstruct:
5603 return m_out.below(
5604 m_callFrame,
5605 m_out.loadPtr(
5606 m_out.absolute(vm().addressOfFTLStackLimit())));
5607
5608 default:
5609 break;
5610 }
5611 }
5612 }
5613
5614 return m_out.booleanFalse;
5615 }
5616
5617 struct ArgumentsLength {
5618 ArgumentsLength()
5619 : isKnown(false)
5620 , known(UINT_MAX)
5621 , value(nullptr)
5622 {
5623 }
5624
5625 bool isKnown;
5626 unsigned known;
5627 LValue value;
5628 };
5629 ArgumentsLength getArgumentsLength(InlineCallFrame* inlineCallFrame)
5630 {
5631 ArgumentsLength length;
5632
5633 if (inlineCallFrame && !inlineCallFrame->isVarargs()) {
5634 length.known = inlineCallFrame->arguments.size() - 1;
5635 length.isKnown = true;
5636 length.value = m_out.constInt32(length.known);
5637 } else {
5638 length.known = UINT_MAX;
5639 length.isKnown = false;
5640
5641 VirtualRegister argumentCountRegister;
5642 if (!inlineCallFrame)
5643 argumentCountRegister = VirtualRegister(JSStack::ArgumentCount);
5644 else
5645 argumentCountRegister = inlineCallFrame->argumentCountRegister;
5646 length.value = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One);
5647 }
5648
5649 return length;
5650 }
5651
5652 ArgumentsLength getArgumentsLength()
5653 {
5654 return getArgumentsLength(m_node->origin.semantic.inlineCallFrame);
5655 }
5656
5657 LValue getCurrentCallee()
5658 {
5659 if (InlineCallFrame* frame = m_node->origin.semantic.inlineCallFrame) {
5660 if (frame->isClosureCall)
5661 return m_out.loadPtr(addressFor(frame->calleeRecovery.virtualRegister()));
5662 return weakPointer(frame->calleeRecovery.constant().asCell());
5663 }
5664 return m_out.loadPtr(addressFor(JSStack::Callee));
5665 }
5666
5667 LValue getArgumentsStart(InlineCallFrame* inlineCallFrame)
5668 {
5669 VirtualRegister start = AssemblyHelpers::argumentsStart(inlineCallFrame);
5670 return addressFor(start).value();
5671 }
5672
5673 LValue getArgumentsStart()
5674 {
5675 return getArgumentsStart(m_node->origin.semantic.inlineCallFrame);
5676 }
5677
5678 template<typename Functor>
5679 void checkStructure(
5680 LValue structureDiscriminant, const FormattedValue& formattedValue, ExitKind exitKind,
5681 const StructureSet& set, const Functor& weakStructureDiscriminant)
5682 {
5683 if (set.size() == 1) {
5684 speculate(
5685 exitKind, formattedValue, 0,
5686 m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set[0])));
5687 return;
5688 }
5689
5690 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("checkStructure continuation"));
5691
5692 LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
5693 for (unsigned i = 0; i < set.size() - 1; ++i) {
5694 LBasicBlock nextStructure = FTL_NEW_BLOCK(m_out, ("checkStructure nextStructure"));
5695 m_out.branch(
5696 m_out.equal(structureDiscriminant, weakStructureDiscriminant(set[i])),
5697 unsure(continuation), unsure(nextStructure));
5698 m_out.appendTo(nextStructure);
5699 }
5700
5701 speculate(
5702 exitKind, formattedValue, 0,
5703 m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set.last())));
5704
5705 m_out.jump(continuation);
5706 m_out.appendTo(continuation, lastNext);
5707 }
5708
5709 LValue numberOrNotCellToInt32(Edge edge, LValue value)
5710 {
5711 LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 int case"));
5712 LBasicBlock notIntCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not int case"));
5713 LBasicBlock doubleCase = 0;
5714 LBasicBlock notNumberCase = 0;
5715 if (edge.useKind() == NotCellUse) {
5716 doubleCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 double case"));
5717 notNumberCase = FTL_NEW_BLOCK(m_out, ("ValueToInt32 not number case"));
5718 }
5719 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ValueToInt32 continuation"));
5720
5721 Vector<ValueFromBlock> results;
5722
5723 m_out.branch(isNotInt32(value), unsure(notIntCase), unsure(intCase));
5724
5725 LBasicBlock lastNext = m_out.appendTo(intCase, notIntCase);
5726 results.append(m_out.anchor(unboxInt32(value)));
5727 m_out.jump(continuation);
5728
5729 if (edge.useKind() == NumberUse) {
5730 m_out.appendTo(notIntCase, continuation);
5731 FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isCellOrMisc(value));
5732 results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
5733 m_out.jump(continuation);
5734 } else {
5735 m_out.appendTo(notIntCase, doubleCase);
5736 m_out.branch(
5737 isCellOrMisc(value, provenType(edge)), unsure(notNumberCase), unsure(doubleCase));
5738
5739 m_out.appendTo(doubleCase, notNumberCase);
5740 results.append(m_out.anchor(doubleToInt32(unboxDouble(value))));
5741 m_out.jump(continuation);
5742
5743 m_out.appendTo(notNumberCase, continuation);
5744
5745 FTL_TYPE_CHECK(jsValueValue(value), edge, ~SpecCell, isCell(value));
5746
5747 LValue specialResult = m_out.select(
5748 m_out.equal(value, m_out.constInt64(JSValue::encode(jsBoolean(true)))),
5749 m_out.int32One, m_out.int32Zero);
5750 results.append(m_out.anchor(specialResult));
5751 m_out.jump(continuation);
5752 }
5753
5754 m_out.appendTo(continuation, lastNext);
5755 return m_out.phi(m_out.int32, results);
5756 }
5757
5758 LValue loadProperty(LValue storage, unsigned identifierNumber, PropertyOffset offset)
5759 {
5760 return m_out.load64(addressOfProperty(storage, identifierNumber, offset));
5761 }
5762
5763 void storeProperty(
5764 LValue value, LValue storage, unsigned identifierNumber, PropertyOffset offset)
5765 {
5766 m_out.store64(value, addressOfProperty(storage, identifierNumber, offset));
5767 }
5768
5769 TypedPointer addressOfProperty(
5770 LValue storage, unsigned identifierNumber, PropertyOffset offset)
5771 {
5772 return m_out.address(
5773 m_heaps.properties[identifierNumber], storage, offsetRelativeToBase(offset));
5774 }
5775
5776 LValue storageForTransition(
5777 LValue object, PropertyOffset offset,
5778 Structure* previousStructure, Structure* nextStructure)
5779 {
5780 if (isInlineOffset(offset))
5781 return object;
5782
5783 if (previousStructure->outOfLineCapacity() == nextStructure->outOfLineCapacity())
5784 return m_out.loadPtr(object, m_heaps.JSObject_butterfly);
5785
5786 LValue result;
5787 if (!previousStructure->outOfLineCapacity())
5788 result = allocatePropertyStorage(object, previousStructure);
5789 else {
5790 result = reallocatePropertyStorage(
5791 object, m_out.loadPtr(object, m_heaps.JSObject_butterfly),
5792 previousStructure, nextStructure);
5793 }
5794
5795 emitStoreBarrier(object);
5796
5797 return result;
5798 }
5799
5800 LValue allocatePropertyStorage(LValue object, Structure* previousStructure)
5801 {
5802 if (previousStructure->couldHaveIndexingHeader()) {
5803 return vmCall(
5804 m_out.operation(
5805 operationReallocateButterflyToHavePropertyStorageWithInitialCapacity),
5806 m_callFrame, object);
5807 }
5808
5809 LValue result = allocatePropertyStorageWithSizeImpl(initialOutOfLineCapacity);
5810 m_out.storePtr(result, object, m_heaps.JSObject_butterfly);
5811 return result;
5812 }
5813
5814 LValue reallocatePropertyStorage(
5815 LValue object, LValue oldStorage, Structure* previous, Structure* next)
5816 {
5817 size_t oldSize = previous->outOfLineCapacity();
5818 size_t newSize = oldSize * outOfLineGrowthFactor;
5819
5820 ASSERT_UNUSED(next, newSize == next->outOfLineCapacity());
5821
5822 if (previous->couldHaveIndexingHeader()) {
5823 LValue newAllocSize = m_out.constIntPtr(newSize);
5824 return vmCall(m_out.operation(operationReallocateButterflyToGrowPropertyStorage), m_callFrame, object, newAllocSize);
5825 }
5826
5827 LValue result = allocatePropertyStorageWithSizeImpl(newSize);
5828
5829 ptrdiff_t headerSize = -sizeof(IndexingHeader) - sizeof(void*);
5830 ptrdiff_t endStorage = headerSize - static_cast<ptrdiff_t>(oldSize * sizeof(JSValue));
5831
5832 for (ptrdiff_t offset = headerSize; offset > endStorage; offset -= sizeof(void*)) {
5833 LValue loaded =
5834 m_out.loadPtr(m_out.address(m_heaps.properties.atAnyNumber(), oldStorage, offset));
5835 m_out.storePtr(loaded, m_out.address(m_heaps.properties.atAnyNumber(), result, offset));
5836 }
5837
5838 m_out.storePtr(result, m_out.address(object, m_heaps.JSObject_butterfly));
5839
5840 return result;
5841 }
5842
5843 LValue allocatePropertyStorageWithSizeImpl(size_t sizeInValues)
5844 {
5845 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorageWithSizeImpl slow path"));
5846 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocatePropertyStorageWithSizeImpl continuation"));
5847
5848 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
5849
5850 LValue endOfStorage = allocateBasicStorageAndGetEnd(
5851 m_out.constIntPtr(sizeInValues * sizeof(JSValue)), slowPath);
5852
5853 ValueFromBlock fastButterfly = m_out.anchor(
5854 m_out.add(m_out.constIntPtr(sizeof(IndexingHeader)), endOfStorage));
5855
5856 m_out.jump(continuation);
5857
5858 m_out.appendTo(slowPath, continuation);
5859
5860 LValue slowButterflyValue;
5861 if (sizeInValues == initialOutOfLineCapacity) {
5862 slowButterflyValue = vmCall(
5863 m_out.operation(operationAllocatePropertyStorageWithInitialCapacity),
5864 m_callFrame);
5865 } else {
5866 slowButterflyValue = vmCall(
5867 m_out.operation(operationAllocatePropertyStorage),
5868 m_callFrame, m_out.constIntPtr(sizeInValues));
5869 }
5870 ValueFromBlock slowButterfly = m_out.anchor(slowButterflyValue);
5871
5872 m_out.jump(continuation);
5873
5874 m_out.appendTo(continuation, lastNext);
5875
5876 return m_out.phi(m_out.intPtr, fastButterfly, slowButterfly);
5877 }
5878
5879 LValue getById(LValue base)
5880 {
5881 auto uid = m_graph.identifiers()[m_node->identifierNumber()];
5882
5883 // Arguments: id, bytes, target, numArgs, args...
5884 unsigned stackmapID = m_stackmapIDs++;
5885
5886 if (Options::verboseCompilation())
5887 dataLog(" Emitting GetById patchpoint with stackmap #", stackmapID, "\n");
5888
5889 LValue call = m_out.call(
5890 m_out.patchpointInt64Intrinsic(),
5891 m_out.constInt64(stackmapID), m_out.constInt32(sizeOfGetById()),
5892 constNull(m_out.ref8), m_out.constInt32(1), base);
5893 setInstructionCallingConvention(call, LLVMAnyRegCallConv);
5894
5895 m_ftlState.getByIds.append(GetByIdDescriptor(stackmapID, m_node->origin.semantic, uid));
5896
5897 return call;
5898 }
5899
5900 TypedPointer baseIndex(IndexedAbstractHeap& heap, LValue storage, LValue index, Edge edge, ptrdiff_t offset = 0)
5901 {
5902 return m_out.baseIndex(
5903 heap, storage, m_out.zeroExtPtr(index), provenValue(edge), offset);
5904 }
5905
5906 void compare(
5907 LIntPredicate intCondition, LRealPredicate realCondition,
5908 S_JITOperation_EJJ helperFunction)
5909 {
5910 if (m_node->isBinaryUseKind(Int32Use)) {
5911 LValue left = lowInt32(m_node->child1());
5912 LValue right = lowInt32(m_node->child2());
5913 setBoolean(m_out.icmp(intCondition, left, right));
5914 return;
5915 }
5916
5917 if (m_node->isBinaryUseKind(Int52RepUse)) {
5918 Int52Kind kind;
5919 LValue left = lowWhicheverInt52(m_node->child1(), kind);
5920 LValue right = lowInt52(m_node->child2(), kind);
5921 setBoolean(m_out.icmp(intCondition, left, right));
5922 return;
5923 }
5924
5925 if (m_node->isBinaryUseKind(DoubleRepUse)) {
5926 LValue left = lowDouble(m_node->child1());
5927 LValue right = lowDouble(m_node->child2());
5928 setBoolean(m_out.fcmp(realCondition, left, right));
5929 return;
5930 }
5931
5932 if (m_node->isBinaryUseKind(UntypedUse)) {
5933 nonSpeculativeCompare(intCondition, helperFunction);
5934 return;
5935 }
5936
5937 DFG_CRASH(m_graph, m_node, "Bad use kinds");
5938 }
5939
5940 void compareEqObjectOrOtherToObject(Edge leftChild, Edge rightChild)
5941 {
5942 LValue rightCell = lowCell(rightChild);
5943 LValue leftValue = lowJSValue(leftChild, ManualOperandSpeculation);
5944
5945 speculateTruthyObject(rightChild, rightCell, SpecObject);
5946
5947 LBasicBlock leftCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left cell case"));
5948 LBasicBlock leftNotCellCase = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject left not cell case"));
5949 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEqObjectOrOtherToObject continuation"));
5950
5951 m_out.branch(
5952 isCell(leftValue, provenType(leftChild)),
5953 unsure(leftCellCase), unsure(leftNotCellCase));
5954
5955 LBasicBlock lastNext = m_out.appendTo(leftCellCase, leftNotCellCase);
5956 speculateTruthyObject(leftChild, leftValue, SpecObject | (~SpecCell));
5957 ValueFromBlock cellResult = m_out.anchor(m_out.equal(rightCell, leftValue));
5958 m_out.jump(continuation);
5959
5960 m_out.appendTo(leftNotCellCase, continuation);
5961 FTL_TYPE_CHECK(
5962 jsValueValue(leftValue), leftChild, SpecOther | SpecCell, isNotOther(leftValue));
5963 ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse);
5964 m_out.jump(continuation);
5965
5966 m_out.appendTo(continuation, lastNext);
5967 setBoolean(m_out.phi(m_out.boolean, cellResult, notCellResult));
5968 }
5969
5970 void speculateTruthyObject(Edge edge, LValue cell, SpeculatedType filter)
5971 {
5972 if (masqueradesAsUndefinedWatchpointIsStillValid()) {
5973 FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell));
5974 return;
5975 }
5976
5977 FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell));
5978 speculate(
5979 BadType, jsValueValue(cell), edge.node(),
5980 m_out.testNonZero8(
5981 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
5982 m_out.constInt8(MasqueradesAsUndefined)));
5983 }
5984
5985 void nonSpeculativeCompare(LIntPredicate intCondition, S_JITOperation_EJJ helperFunction)
5986 {
5987 LValue left = lowJSValue(m_node->child1());
5988 LValue right = lowJSValue(m_node->child2());
5989
5990 LBasicBlock leftIsInt = FTL_NEW_BLOCK(m_out, ("CompareEq untyped left is int"));
5991 LBasicBlock fastPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped fast path"));
5992 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("CompareEq untyped slow path"));
5993 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("CompareEq untyped continuation"));
5994
5995 m_out.branch(isNotInt32(left), rarely(slowPath), usually(leftIsInt));
5996
5997 LBasicBlock lastNext = m_out.appendTo(leftIsInt, fastPath);
5998 m_out.branch(isNotInt32(right), rarely(slowPath), usually(fastPath));
5999
6000 m_out.appendTo(fastPath, slowPath);
6001 ValueFromBlock fastResult = m_out.anchor(
6002 m_out.icmp(intCondition, unboxInt32(left), unboxInt32(right)));
6003 m_out.jump(continuation);
6004
6005 m_out.appendTo(slowPath, continuation);
6006 ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall(
6007 m_out.operation(helperFunction), m_callFrame, left, right)));
6008 m_out.jump(continuation);
6009
6010 m_out.appendTo(continuation, lastNext);
6011 setBoolean(m_out.phi(m_out.boolean, fastResult, slowResult));
6012 }
6013
6014 LValue allocateCell(LValue allocator, LBasicBlock slowPath)
6015 {
6016 LBasicBlock success = FTL_NEW_BLOCK(m_out, ("object allocation success"));
6017
6018 LValue result = m_out.loadPtr(
6019 allocator, m_heaps.MarkedAllocator_freeListHead);
6020
6021 m_out.branch(m_out.notNull(result), usually(success), rarely(slowPath));
6022
6023 m_out.appendTo(success);
6024
6025 m_out.storePtr(
6026 m_out.loadPtr(result, m_heaps.JSCell_freeListNext),
6027 allocator, m_heaps.MarkedAllocator_freeListHead);
6028
6029 return result;
6030 }
6031
6032 void storeStructure(LValue object, Structure* structure)
6033 {
6034 m_out.store32(m_out.constInt32(structure->id()), object, m_heaps.JSCell_structureID);
6035 m_out.store32(
6036 m_out.constInt32(structure->objectInitializationBlob()),
6037 object, m_heaps.JSCell_usefulBytes);
6038 }
6039
6040 LValue allocateCell(LValue allocator, Structure* structure, LBasicBlock slowPath)
6041 {
6042 LValue result = allocateCell(allocator, slowPath);
6043 storeStructure(result, structure);
6044 return result;
6045 }
6046
6047 LValue allocateObject(
6048 LValue allocator, Structure* structure, LValue butterfly, LBasicBlock slowPath)
6049 {
6050 LValue result = allocateCell(allocator, structure, slowPath);
6051 m_out.storePtr(butterfly, result, m_heaps.JSObject_butterfly);
6052 return result;
6053 }
6054
6055 template<typename ClassType>
6056 LValue allocateObject(
6057 size_t size, Structure* structure, LValue butterfly, LBasicBlock slowPath)
6058 {
6059 MarkedAllocator* allocator = &vm().heap.allocatorForObjectOfType<ClassType>(size);
6060 return allocateObject(m_out.constIntPtr(allocator), structure, butterfly, slowPath);
6061 }
6062
6063 template<typename ClassType>
6064 LValue allocateObject(Structure* structure, LValue butterfly, LBasicBlock slowPath)
6065 {
6066 return allocateObject<ClassType>(
6067 ClassType::allocationSize(0), structure, butterfly, slowPath);
6068 }
6069
6070 template<typename ClassType>
6071 LValue allocateVariableSizedObject(
6072 LValue size, Structure* structure, LValue butterfly, LBasicBlock slowPath)
6073 {
6074 static_assert(!(MarkedSpace::preciseStep & (MarkedSpace::preciseStep - 1)), "MarkedSpace::preciseStep must be a power of two.");
6075 static_assert(!(MarkedSpace::impreciseStep & (MarkedSpace::impreciseStep - 1)), "MarkedSpace::impreciseStep must be a power of two.");
6076
6077 LValue subspace = m_out.constIntPtr(&vm().heap.subspaceForObjectOfType<ClassType>());
6078
6079 LBasicBlock smallCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject small case"));
6080 LBasicBlock largeOrOversizeCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject large or oversize case"));
6081 LBasicBlock largeCaseBlock = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject large case"));
6082 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocateVariableSizedObject continuation"));
6083
6084 LValue uproundedSize = m_out.add(size, m_out.constInt32(MarkedSpace::preciseStep - 1));
6085 LValue isSmall = m_out.below(uproundedSize, m_out.constInt32(MarkedSpace::preciseCutoff));
6086 m_out.branch(isSmall, unsure(smallCaseBlock), unsure(largeOrOversizeCaseBlock));
6087
6088 LBasicBlock lastNext = m_out.appendTo(smallCaseBlock, largeOrOversizeCaseBlock);
6089 TypedPointer address = m_out.baseIndex(
6090 m_heaps.MarkedSpace_Subspace_preciseAllocators, subspace,
6091 m_out.zeroExtPtr(m_out.lShr(uproundedSize, m_out.constInt32(getLSBSet(MarkedSpace::preciseStep)))));
6092 ValueFromBlock smallAllocator = m_out.anchor(address.value());
6093 m_out.jump(continuation);
6094
6095 m_out.appendTo(largeOrOversizeCaseBlock, largeCaseBlock);
6096 m_out.branch(
6097 m_out.below(uproundedSize, m_out.constInt32(MarkedSpace::impreciseCutoff)),
6098 usually(largeCaseBlock), rarely(slowPath));
6099
6100 m_out.appendTo(largeCaseBlock, continuation);
6101 address = m_out.baseIndex(
6102 m_heaps.MarkedSpace_Subspace_impreciseAllocators, subspace,
6103 m_out.zeroExtPtr(m_out.lShr(uproundedSize, m_out.constInt32(getLSBSet(MarkedSpace::impreciseStep)))));
6104 ValueFromBlock largeAllocator = m_out.anchor(address.value());
6105 m_out.jump(continuation);
6106
6107 m_out.appendTo(continuation, lastNext);
6108 LValue allocator = m_out.phi(m_out.intPtr, smallAllocator, largeAllocator);
6109
6110 return allocateObject(allocator, structure, butterfly, slowPath);
6111 }
6112
6113 // Returns a pointer to the end of the allocation.
6114 LValue allocateBasicStorageAndGetEnd(LValue size, LBasicBlock slowPath)
6115 {
6116 CopiedAllocator& allocator = vm().heap.storageAllocator();
6117
6118 LBasicBlock success = FTL_NEW_BLOCK(m_out, ("storage allocation success"));
6119
6120 LValue remaining = m_out.loadPtr(m_out.absolute(&allocator.m_currentRemaining));
6121 LValue newRemaining = m_out.sub(remaining, size);
6122
6123 m_out.branch(
6124 m_out.lessThan(newRemaining, m_out.intPtrZero),
6125 rarely(slowPath), usually(success));
6126
6127 m_out.appendTo(success);
6128
6129 m_out.storePtr(newRemaining, m_out.absolute(&allocator.m_currentRemaining));
6130 return m_out.sub(
6131 m_out.loadPtr(m_out.absolute(&allocator.m_currentPayloadEnd)), newRemaining);
6132 }
6133
6134 LValue allocateObject(Structure* structure)
6135 {
6136 size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity());
6137 MarkedAllocator* allocator = &vm().heap.allocatorForObjectWithoutDestructor(allocationSize);
6138
6139 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("allocateObject slow path"));
6140 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("allocateObject continuation"));
6141
6142 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
6143
6144 ValueFromBlock fastResult = m_out.anchor(allocateObject(
6145 m_out.constIntPtr(allocator), structure, m_out.intPtrZero, slowPath));
6146
6147 m_out.jump(continuation);
6148
6149 m_out.appendTo(slowPath, continuation);
6150
6151 ValueFromBlock slowResult = m_out.anchor(vmCall(
6152 m_out.operation(operationNewObject), m_callFrame, m_out.constIntPtr(structure)));
6153 m_out.jump(continuation);
6154
6155 m_out.appendTo(continuation, lastNext);
6156 return m_out.phi(m_out.intPtr, fastResult, slowResult);
6157 }
6158
6159 struct ArrayValues {
6160 ArrayValues()
6161 : array(0)
6162 , butterfly(0)
6163 {
6164 }
6165
6166 ArrayValues(LValue array, LValue butterfly)
6167 : array(array)
6168 , butterfly(butterfly)
6169 {
6170 }
6171
6172 LValue array;
6173 LValue butterfly;
6174 };
6175 ArrayValues allocateJSArray(
6176 Structure* structure, unsigned numElements, LBasicBlock slowPath)
6177 {
6178 ASSERT(
6179 hasUndecided(structure->indexingType())
6180 || hasInt32(structure->indexingType())
6181 || hasDouble(structure->indexingType())
6182 || hasContiguous(structure->indexingType()));
6183
6184 unsigned vectorLength = std::max(BASE_VECTOR_LEN, numElements);
6185
6186 LValue endOfStorage = allocateBasicStorageAndGetEnd(
6187 m_out.constIntPtr(sizeof(JSValue) * vectorLength + sizeof(IndexingHeader)),
6188 slowPath);
6189
6190 LValue butterfly = m_out.sub(
6191 endOfStorage, m_out.constIntPtr(sizeof(JSValue) * vectorLength));
6192
6193 LValue object = allocateObject<JSArray>(
6194 structure, butterfly, slowPath);
6195
6196 m_out.store32(m_out.constInt32(numElements), butterfly, m_heaps.Butterfly_publicLength);
6197 m_out.store32(m_out.constInt32(vectorLength), butterfly, m_heaps.Butterfly_vectorLength);
6198
6199 if (hasDouble(structure->indexingType())) {
6200 for (unsigned i = numElements; i < vectorLength; ++i) {
6201 m_out.store64(
6202 m_out.constInt64(bitwise_cast<int64_t>(PNaN)),
6203 butterfly, m_heaps.indexedDoubleProperties[i]);
6204 }
6205 }
6206
6207 return ArrayValues(object, butterfly);
6208 }
6209
6210 ArrayValues allocateJSArray(Structure* structure, unsigned numElements)
6211 {
6212 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("JSArray allocation slow path"));
6213 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("JSArray allocation continuation"));
6214
6215 LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath);
6216
6217 ArrayValues fastValues = allocateJSArray(structure, numElements, slowPath);
6218 ValueFromBlock fastArray = m_out.anchor(fastValues.array);
6219 ValueFromBlock fastButterfly = m_out.anchor(fastValues.butterfly);
6220
6221 m_out.jump(continuation);
6222
6223 m_out.appendTo(slowPath, continuation);
6224
6225 ValueFromBlock slowArray = m_out.anchor(vmCall(
6226 m_out.operation(operationNewArrayWithSize), m_callFrame,
6227 m_out.constIntPtr(structure), m_out.constInt32(numElements)));
6228 ValueFromBlock slowButterfly = m_out.anchor(
6229 m_out.loadPtr(slowArray.value(), m_heaps.JSObject_butterfly));
6230
6231 m_out.jump(continuation);
6232
6233 m_out.appendTo(continuation, lastNext);
6234
6235 return ArrayValues(
6236 m_out.phi(m_out.intPtr, fastArray, slowArray),
6237 m_out.phi(m_out.intPtr, fastButterfly, slowButterfly));
6238 }
6239
6240 LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode, LValue base)
6241 {
6242 JSArrayBufferView* view = m_graph.tryGetFoldableView(provenValue(baseEdge), arrayMode);
6243 if (view)
6244 return m_out.constInt32(view->length());
6245 return m_out.load32NonNegative(base, m_heaps.JSArrayBufferView_length);
6246 }
6247
6248 LValue typedArrayLength(Edge baseEdge, ArrayMode arrayMode)
6249 {
6250 return typedArrayLength(baseEdge, arrayMode, lowCell(baseEdge));
6251 }
6252
6253 LValue boolify(Edge edge)
6254 {
6255 switch (edge.useKind()) {
6256 case BooleanUse:
6257 return lowBoolean(edge);
6258 case Int32Use:
6259 return m_out.notZero32(lowInt32(edge));
6260 case DoubleRepUse:
6261 return m_out.doubleNotEqual(lowDouble(edge), m_out.doubleZero);
6262 case ObjectOrOtherUse:
6263 return m_out.bitNot(
6264 equalNullOrUndefined(
6265 edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined,
6266 ManualOperandSpeculation));
6267 case StringUse: {
6268 LValue stringValue = lowString(edge);
6269 LValue length = m_out.load32NonNegative(stringValue, m_heaps.JSString_length);
6270 return m_out.notEqual(length, m_out.int32Zero);
6271 }
6272 case UntypedUse: {
6273 LValue value = lowJSValue(edge);
6274
6275 // Implements the following control flow structure:
6276 // if (value is cell) {
6277 // if (value is string)
6278 // result = !!value->length
6279 // else {
6280 // do evil things for masquerades-as-undefined
6281 // result = true
6282 // }
6283 // } else if (value is int32) {
6284 // result = !!unboxInt32(value)
6285 // } else if (value is number) {
6286 // result = !!unboxDouble(value)
6287 // } else {
6288 // result = value == jsTrue
6289 // }
6290
6291 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped cell case"));
6292 LBasicBlock stringCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped string case"));
6293 LBasicBlock notStringCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not string case"));
6294 LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not cell case"));
6295 LBasicBlock int32Case = FTL_NEW_BLOCK(m_out, ("Boolify untyped int32 case"));
6296 LBasicBlock notInt32Case = FTL_NEW_BLOCK(m_out, ("Boolify untyped not int32 case"));
6297 LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped double case"));
6298 LBasicBlock notDoubleCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped not double case"));
6299 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Boolify untyped continuation"));
6300
6301 Vector<ValueFromBlock> results;
6302
6303 m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase));
6304
6305 LBasicBlock lastNext = m_out.appendTo(cellCase, stringCase);
6306 m_out.branch(
6307 isString(value, provenType(edge) & SpecCell),
6308 unsure(stringCase), unsure(notStringCase));
6309
6310 m_out.appendTo(stringCase, notStringCase);
6311 LValue nonEmptyString = m_out.notZero32(
6312 m_out.load32NonNegative(value, m_heaps.JSString_length));
6313 results.append(m_out.anchor(nonEmptyString));
6314 m_out.jump(continuation);
6315
6316 m_out.appendTo(notStringCase, notCellCase);
6317 LValue isTruthyObject;
6318 if (masqueradesAsUndefinedWatchpointIsStillValid())
6319 isTruthyObject = m_out.booleanTrue;
6320 else {
6321 LBasicBlock masqueradesCase = FTL_NEW_BLOCK(m_out, ("Boolify untyped masquerades case"));
6322
6323 results.append(m_out.anchor(m_out.booleanTrue));
6324
6325 m_out.branch(
6326 m_out.testIsZero8(
6327 m_out.load8(value, m_heaps.JSCell_typeInfoFlags),
6328 m_out.constInt8(MasqueradesAsUndefined)),
6329 usually(continuation), rarely(masqueradesCase));
6330
6331 m_out.appendTo(masqueradesCase);
6332
6333 isTruthyObject = m_out.notEqual(
6334 m_out.constIntPtr(m_graph.globalObjectFor(m_node->origin.semantic)),
6335 m_out.loadPtr(loadStructure(value), m_heaps.Structure_globalObject));
6336 }
6337 results.append(m_out.anchor(isTruthyObject));
6338 m_out.jump(continuation);
6339
6340 m_out.appendTo(notCellCase, int32Case);
6341 m_out.branch(
6342 isInt32(value, provenType(edge) & ~SpecCell),
6343 unsure(int32Case), unsure(notInt32Case));
6344
6345 m_out.appendTo(int32Case, notInt32Case);
6346 results.append(m_out.anchor(m_out.notZero32(unboxInt32(value))));
6347 m_out.jump(continuation);
6348
6349 m_out.appendTo(notInt32Case, doubleCase);
6350 m_out.branch(
6351 isNumber(value, provenType(edge) & ~SpecCell),
6352 unsure(doubleCase), unsure(notDoubleCase));
6353
6354 m_out.appendTo(doubleCase, notDoubleCase);
6355 // Note that doubleNotEqual() really means not-equal-and-ordered. It will return false
6356 // if value is NaN.
6357 LValue doubleIsTruthy = m_out.doubleNotEqual(
6358 unboxDouble(value), m_out.constDouble(0));
6359 results.append(m_out.anchor(doubleIsTruthy));
6360 m_out.jump(continuation);
6361
6362 m_out.appendTo(notDoubleCase, continuation);
6363 LValue miscIsTruthy = m_out.equal(
6364 value, m_out.constInt64(JSValue::encode(jsBoolean(true))));
6365 results.append(m_out.anchor(miscIsTruthy));
6366 m_out.jump(continuation);
6367
6368 m_out.appendTo(continuation, lastNext);
6369 return m_out.phi(m_out.boolean, results);
6370 }
6371 default:
6372 DFG_CRASH(m_graph, m_node, "Bad use kind");
6373 return 0;
6374 }
6375 }
6376
6377 enum StringOrObjectMode {
6378 AllCellsAreFalse,
6379 CellCaseSpeculatesObject
6380 };
6381 enum EqualNullOrUndefinedMode {
6382 EqualNull,
6383 EqualUndefined,
6384 EqualNullOrUndefined,
6385 SpeculateNullOrUndefined
6386 };
6387 LValue equalNullOrUndefined(
6388 Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode,
6389 OperandSpeculationMode operandMode = AutomaticOperandSpeculation)
6390 {
6391 bool validWatchpoint = masqueradesAsUndefinedWatchpointIsStillValid();
6392
6393 LValue value = lowJSValue(edge, operandMode);
6394
6395 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined cell case"));
6396 LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined primitive case"));
6397 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined continuation"));
6398
6399 m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase));
6400
6401 LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
6402
6403 Vector<ValueFromBlock, 3> results;
6404
6405 switch (cellMode) {
6406 case AllCellsAreFalse:
6407 break;
6408 case CellCaseSpeculatesObject:
6409 FTL_TYPE_CHECK(
6410 jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value));
6411 break;
6412 }
6413
6414 if (validWatchpoint) {
6415 results.append(m_out.anchor(m_out.booleanFalse));
6416 m_out.jump(continuation);
6417 } else {
6418 LBasicBlock masqueradesCase =
6419 FTL_NEW_BLOCK(m_out, ("EqualNullOrUndefined masquerades case"));
6420
6421 results.append(m_out.anchor(m_out.booleanFalse));
6422
6423 m_out.branch(
6424 m_out.testNonZero8(
6425 m_out.load8(value, m_heaps.JSCell_typeInfoFlags),
6426 m_out.constInt8(MasqueradesAsUndefined)),
6427 rarely(masqueradesCase), usually(continuation));
6428
6429 m_out.appendTo(masqueradesCase, primitiveCase);
6430
6431 LValue structure = loadStructure(value);
6432
6433 results.append(m_out.anchor(
6434 m_out.equal(
6435 m_out.constIntPtr(m_graph.globalObjectFor(m_node->origin.semantic)),
6436 m_out.loadPtr(structure, m_heaps.Structure_globalObject))));
6437 m_out.jump(continuation);
6438 }
6439
6440 m_out.appendTo(primitiveCase, continuation);
6441
6442 LValue primitiveResult;
6443 switch (primitiveMode) {
6444 case EqualNull:
6445 primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull));
6446 break;
6447 case EqualUndefined:
6448 primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined));
6449 break;
6450 case EqualNullOrUndefined:
6451 primitiveResult = isOther(value, provenType(edge));
6452 break;
6453 case SpeculateNullOrUndefined:
6454 FTL_TYPE_CHECK(
6455 jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value));
6456 primitiveResult = m_out.booleanTrue;
6457 break;
6458 }
6459 results.append(m_out.anchor(primitiveResult));
6460 m_out.jump(continuation);
6461
6462 m_out.appendTo(continuation, lastNext);
6463
6464 return m_out.phi(m_out.boolean, results);
6465 }
6466
6467 template<typename FunctionType>
6468 void contiguousPutByValOutOfBounds(
6469 FunctionType slowPathFunction, LValue base, LValue storage, LValue index, LValue value,
6470 LBasicBlock continuation)
6471 {
6472 LValue isNotInBounds = m_out.aboveOrEqual(
6473 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength));
6474 if (!m_node->arrayMode().isInBounds()) {
6475 LBasicBlock notInBoundsCase =
6476 FTL_NEW_BLOCK(m_out, ("PutByVal not in bounds"));
6477 LBasicBlock performStore =
6478 FTL_NEW_BLOCK(m_out, ("PutByVal perform store"));
6479
6480 m_out.branch(isNotInBounds, unsure(notInBoundsCase), unsure(performStore));
6481
6482 LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore);
6483
6484 LValue isOutOfBounds = m_out.aboveOrEqual(
6485 index, m_out.load32NonNegative(storage, m_heaps.Butterfly_vectorLength));
6486
6487 if (!m_node->arrayMode().isOutOfBounds())
6488 speculate(OutOfBounds, noValue(), 0, isOutOfBounds);
6489 else {
6490 LBasicBlock outOfBoundsCase =
6491 FTL_NEW_BLOCK(m_out, ("PutByVal out of bounds"));
6492 LBasicBlock holeCase =
6493 FTL_NEW_BLOCK(m_out, ("PutByVal hole case"));
6494
6495 m_out.branch(isOutOfBounds, unsure(outOfBoundsCase), unsure(holeCase));
6496
6497 LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase);
6498
6499 vmCall(
6500 m_out.operation(slowPathFunction),
6501 m_callFrame, base, index, value);
6502
6503 m_out.jump(continuation);
6504
6505 m_out.appendTo(holeCase, innerLastNext);
6506 }
6507
6508 m_out.store32(
6509 m_out.add(index, m_out.int32One),
6510 storage, m_heaps.Butterfly_publicLength);
6511
6512 m_out.jump(performStore);
6513 m_out.appendTo(performStore, lastNext);
6514 }
6515 }
6516
6517 void buildSwitch(SwitchData* data, LType type, LValue switchValue)
6518 {
6519 Vector<SwitchCase> cases;
6520 for (unsigned i = 0; i < data->cases.size(); ++i) {
6521 cases.append(SwitchCase(
6522 constInt(type, data->cases[i].value.switchLookupValue(data->kind)),
6523 lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count)));
6524 }
6525
6526 m_out.switchInstruction(
6527 switchValue, cases,
6528 lowBlock(data->fallThrough.block), Weight(data->fallThrough.count));
6529 }
6530
6531 void switchString(SwitchData* data, LValue string)
6532 {
6533 bool canDoBinarySwitch = true;
6534 unsigned totalLength = 0;
6535
6536 for (DFG::SwitchCase myCase : data->cases) {
6537 StringImpl* string = myCase.value.stringImpl();
6538 if (!string->is8Bit()) {
6539 canDoBinarySwitch = false;
6540 break;
6541 }
6542 if (string->length() > Options::maximumBinaryStringSwitchCaseLength()) {
6543 canDoBinarySwitch = false;
6544 break;
6545 }
6546 totalLength += string->length();
6547 }
6548
6549 if (!canDoBinarySwitch || totalLength > Options::maximumBinaryStringSwitchTotalLength()) {
6550 switchStringSlow(data, string);
6551 return;
6552 }
6553
6554 LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
6555 LValue length = m_out.load32(string, m_heaps.JSString_length);
6556
6557 LBasicBlock hasImplBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString has impl case"));
6558 LBasicBlock is8BitBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString is 8 bit case"));
6559 LBasicBlock slowBlock = FTL_NEW_BLOCK(m_out, ("Switch/SwitchString slow case"));
6560
6561 m_out.branch(m_out.isNull(stringImpl), unsure(slowBlock), unsure(hasImplBlock));
6562
6563 LBasicBlock lastNext = m_out.appendTo(hasImplBlock, is8BitBlock);
6564
6565 m_out.branch(
6566 m_out.testIsZero32(
6567 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
6568 m_out.constInt32(StringImpl::flagIs8Bit())),
6569 unsure(slowBlock), unsure(is8BitBlock));
6570
6571 m_out.appendTo(is8BitBlock, slowBlock);
6572
6573 LValue buffer = m_out.loadPtr(stringImpl, m_heaps.StringImpl_data);
6574
6575 // FIXME: We should propagate branch weight data to the cases of this switch.
6576 // https://bugs.webkit.org/show_bug.cgi?id=144368
6577
6578 Vector<StringSwitchCase> cases;
6579 for (DFG::SwitchCase myCase : data->cases)
6580 cases.append(StringSwitchCase(myCase.value.stringImpl(), lowBlock(myCase.target.block)));
6581 std::sort(cases.begin(), cases.end());
6582 switchStringRecurse(data, buffer, length, cases, 0, 0, cases.size(), 0, false);
6583
6584 m_out.appendTo(slowBlock, lastNext);
6585 switchStringSlow(data, string);
6586 }
6587
6588 // The code for string switching is based closely on the same code in the DFG backend. While it
6589 // would be nice to reduce the amount of similar-looking code, it seems like this is one of
6590 // those algorithms where factoring out the common bits would result in more code than just
6591 // duplicating.
6592
6593 struct StringSwitchCase {
6594 StringSwitchCase() { }
6595
6596 StringSwitchCase(StringImpl* string, LBasicBlock target)
6597 : string(string)
6598 , target(target)
6599 {
6600 }
6601
6602 bool operator<(const StringSwitchCase& other) const
6603 {
6604 return stringLessThan(*string, *other.string);
6605 }
6606
6607 StringImpl* string;
6608 LBasicBlock target;
6609 };
6610
6611 struct CharacterCase {
6612 CharacterCase()
6613 : character(0)
6614 , begin(0)
6615 , end(0)
6616 {
6617 }
6618
6619 CharacterCase(LChar character, unsigned begin, unsigned end)
6620 : character(character)
6621 , begin(begin)
6622 , end(end)
6623 {
6624 }
6625
6626 bool operator<(const CharacterCase& other) const
6627 {
6628 return character < other.character;
6629 }
6630
6631 LChar character;
6632 unsigned begin;
6633 unsigned end;
6634 };
6635
6636 void switchStringRecurse(
6637 SwitchData* data, LValue buffer, LValue length, const Vector<StringSwitchCase>& cases,
6638 unsigned numChecked, unsigned begin, unsigned end, unsigned alreadyCheckedLength,
6639 unsigned checkedExactLength)
6640 {
6641 LBasicBlock fallThrough = lowBlock(data->fallThrough.block);
6642
6643 if (begin == end) {
6644 m_out.jump(fallThrough);
6645 return;
6646 }
6647
6648 unsigned minLength = cases[begin].string->length();
6649 unsigned commonChars = minLength;
6650 bool allLengthsEqual = true;
6651 for (unsigned i = begin + 1; i < end; ++i) {
6652 unsigned myCommonChars = numChecked;
6653 unsigned limit = std::min(cases[begin].string->length(), cases[i].string->length());
6654 for (unsigned j = numChecked; j < limit; ++j) {
6655 if (cases[begin].string->at(j) != cases[i].string->at(j))
6656 break;
6657 myCommonChars++;
6658 }
6659 commonChars = std::min(commonChars, myCommonChars);
6660 if (minLength != cases[i].string->length())
6661 allLengthsEqual = false;
6662 minLength = std::min(minLength, cases[i].string->length());
6663 }
6664
6665 if (checkedExactLength) {
6666 DFG_ASSERT(m_graph, m_node, alreadyCheckedLength == minLength);
6667 DFG_ASSERT(m_graph, m_node, allLengthsEqual);
6668 }
6669
6670 DFG_ASSERT(m_graph, m_node, minLength >= commonChars);
6671
6672 if (!allLengthsEqual && alreadyCheckedLength < minLength)
6673 m_out.check(m_out.below(length, m_out.constInt32(minLength)), unsure(fallThrough));
6674 if (allLengthsEqual && (alreadyCheckedLength < minLength || !checkedExactLength))
6675 m_out.check(m_out.notEqual(length, m_out.constInt32(minLength)), unsure(fallThrough));
6676
6677 for (unsigned i = numChecked; i < commonChars; ++i) {
6678 m_out.check(
6679 m_out.notEqual(
6680 m_out.load8(buffer, m_heaps.characters8[i]),
6681 m_out.constInt8(cases[begin].string->at(i))),
6682 unsure(fallThrough));
6683 }
6684
6685 if (minLength == commonChars) {
6686 // This is the case where one of the cases is a prefix of all of the other cases.
6687 // We've already checked that the input string is a prefix of all of the cases,
6688 // so we just check length to jump to that case.
6689
6690 DFG_ASSERT(m_graph, m_node, cases[begin].string->length() == commonChars);
6691 for (unsigned i = begin + 1; i < end; ++i)
6692 DFG_ASSERT(m_graph, m_node, cases[i].string->length() > commonChars);
6693
6694 if (allLengthsEqual) {
6695 DFG_ASSERT(m_graph, m_node, end == begin + 1);
6696 m_out.jump(cases[begin].target);
6697 return;
6698 }
6699
6700 m_out.check(
6701 m_out.equal(length, m_out.constInt32(commonChars)),
6702 unsure(cases[begin].target));
6703
6704 // We've checked if the length is >= minLength, and then we checked if the length is
6705 // == commonChars. We get to this point if it is >= minLength but not == commonChars.
6706 // Hence we know that it now must be > minLength, i.e. that it's >= minLength + 1.
6707 switchStringRecurse(
6708 data, buffer, length, cases, commonChars, begin + 1, end, minLength + 1, false);
6709 return;
6710 }
6711
6712 // At this point we know that the string is longer than commonChars, and we've only verified
6713 // commonChars. Use a binary switch on the next unchecked character, i.e.
6714 // string[commonChars].
6715
6716 DFG_ASSERT(m_graph, m_node, end >= begin + 2);
6717
6718 LValue uncheckedChar = m_out.load8(buffer, m_heaps.characters8[commonChars]);
6719
6720 Vector<CharacterCase> characterCases;
6721 CharacterCase currentCase(cases[begin].string->at(commonChars), begin, begin + 1);
6722 for (unsigned i = begin + 1; i < end; ++i) {
6723 LChar currentChar = cases[i].string->at(commonChars);
6724 if (currentChar != currentCase.character) {
6725 currentCase.end = i;
6726 characterCases.append(currentCase);
6727 currentCase = CharacterCase(currentChar, i, i + 1);
6728 } else
6729 currentCase.end = i + 1;
6730 }
6731 characterCases.append(currentCase);
6732
6733 Vector<LBasicBlock> characterBlocks;
6734 for (CharacterCase& myCase : characterCases)
6735 characterBlocks.append(FTL_NEW_BLOCK(m_out, ("Switch/SwitchString case for ", myCase.character, " at index ", commonChars)));
6736
6737 Vector<SwitchCase> switchCases;
6738 for (unsigned i = 0; i < characterCases.size(); ++i) {
6739 if (i)
6740 DFG_ASSERT(m_graph, m_node, characterCases[i - 1].character < characterCases[i].character);
6741 switchCases.append(SwitchCase(
6742 m_out.constInt8(characterCases[i].character), characterBlocks[i], Weight()));
6743 }
6744 m_out.switchInstruction(uncheckedChar, switchCases, fallThrough, Weight());
6745
6746 LBasicBlock lastNext = m_out.m_nextBlock;
6747 characterBlocks.append(lastNext); // Makes it convenient to set nextBlock.
6748 for (unsigned i = 0; i < characterCases.size(); ++i) {
6749 m_out.appendTo(characterBlocks[i], characterBlocks[i + 1]);
6750 switchStringRecurse(
6751 data, buffer, length, cases, commonChars + 1,
6752 characterCases[i].begin, characterCases[i].end, minLength, allLengthsEqual);
6753 }
6754
6755 DFG_ASSERT(m_graph, m_node, m_out.m_nextBlock == lastNext);
6756 }
6757
6758 void switchStringSlow(SwitchData* data, LValue string)
6759 {
6760 // FIXME: We ought to be able to use computed gotos here. We would save the labels of the
6761 // blocks we want to jump to, and then request their addresses after compilation completes.
6762 // https://bugs.webkit.org/show_bug.cgi?id=144369
6763
6764 LValue branchOffset = vmCall(
6765 m_out.operation(operationSwitchStringAndGetBranchOffset),
6766 m_callFrame, m_out.constIntPtr(data->switchTableIndex), string);
6767
6768 StringJumpTable& table = codeBlock()->stringSwitchJumpTable(data->switchTableIndex);
6769
6770 Vector<SwitchCase> cases;
6771 std::unordered_set<int32_t> alreadyHandled; // These may be negative, or zero, or probably other stuff, too. We don't want to mess with HashSet's corner cases and we don't really care about throughput here.
6772 for (unsigned i = 0; i < data->cases.size(); ++i) {
6773 // FIXME: The fact that we're using the bytecode's switch table means that the
6774 // following DFG IR transformation would be invalid.
6775 //
6776 // Original code:
6777 // switch (v) {
6778 // case "foo":
6779 // case "bar":
6780 // things();
6781 // break;
6782 // default:
6783 // break;
6784 // }
6785 //
6786 // New code:
6787 // switch (v) {
6788 // case "foo":
6789 // instrumentFoo();
6790 // goto _things;
6791 // case "bar":
6792 // instrumentBar();
6793 // _things:
6794 // things();
6795 // break;
6796 // default:
6797 // break;
6798 // }
6799 //
6800 // Luckily, we don't currently do any such transformation. But it's kind of silly that
6801 // this is an issue.
6802 // https://bugs.webkit.org/show_bug.cgi?id=144635
6803
6804 DFG::SwitchCase myCase = data->cases[i];
6805 StringJumpTable::StringOffsetTable::iterator iter =
6806 table.offsetTable.find(myCase.value.stringImpl());
6807 DFG_ASSERT(m_graph, m_node, iter != table.offsetTable.end());
6808
6809 if (!alreadyHandled.insert(iter->value.branchOffset).second)
6810 continue;
6811
6812 cases.append(SwitchCase(
6813 m_out.constInt32(iter->value.branchOffset),
6814 lowBlock(myCase.target.block), Weight(myCase.target.count)));
6815 }
6816
6817 m_out.switchInstruction(
6818 branchOffset, cases, lowBlock(data->fallThrough.block),
6819 Weight(data->fallThrough.count));
6820 }
6821
6822 // Calls the functor at the point of code generation where we know what the result type is.
6823 // You can emit whatever code you like at that point. Expects you to terminate the basic block.
6824 // When buildTypeOf() returns, it will have terminated all basic blocks that it created. So, if
6825 // you aren't using this as the terminator of a high-level block, you should create your own
6826 // contination and set it as the nextBlock (m_out.insertNewBlocksBefore(continuation)) before
6827 // calling this. For example:
6828 //
6829 // LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("My continuation"));
6830 // LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation);
6831 // buildTypeOf(
6832 // child, value,
6833 // [&] (TypeofType type) {
6834 // do things;
6835 // m_out.jump(continuation);
6836 // });
6837 // m_out.appendTo(continuation, lastNext);
6838 template<typename Functor>
6839 void buildTypeOf(Edge child, LValue value, const Functor& functor)
6840 {
6841 JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic);
6842
6843 // Implements the following branching structure:
6844 //
6845 // if (is cell) {
6846 // if (is object) {
6847 // if (is function) {
6848 // return function;
6849 // } else if (doesn't have call trap and doesn't masquerade as undefined) {
6850 // return object
6851 // } else {
6852 // return slowPath();
6853 // }
6854 // } else if (is string) {
6855 // return string
6856 // } else {
6857 // return symbol
6858 // }
6859 // } else if (is number) {
6860 // return number
6861 // } else if (is null) {
6862 // return object
6863 // } else if (is boolean) {
6864 // return boolean
6865 // } else {
6866 // return undefined
6867 // }
6868
6869 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf cell case"));
6870 LBasicBlock objectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf object case"));
6871 LBasicBlock functionCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf function case"));
6872 LBasicBlock notFunctionCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not function case"));
6873 LBasicBlock reallyObjectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf really object case"));
6874 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("buildTypeOf slow path"));
6875 LBasicBlock unreachable = FTL_NEW_BLOCK(m_out, ("buildTypeOf unreachable"));
6876 LBasicBlock notObjectCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not object case"));
6877 LBasicBlock stringCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf string case"));
6878 LBasicBlock symbolCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf symbol case"));
6879 LBasicBlock notCellCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not cell case"));
6880 LBasicBlock numberCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf number case"));
6881 LBasicBlock notNumberCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not number case"));
6882 LBasicBlock notNullCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf not null case"));
6883 LBasicBlock booleanCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf boolean case"));
6884 LBasicBlock undefinedCase = FTL_NEW_BLOCK(m_out, ("buildTypeOf undefined case"));
6885
6886 m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase));
6887
6888 LBasicBlock lastNext = m_out.appendTo(cellCase, objectCase);
6889 m_out.branch(isObject(value, provenType(child)), unsure(objectCase), unsure(notObjectCase));
6890
6891 m_out.appendTo(objectCase, functionCase);
6892 m_out.branch(
6893 isFunction(value, provenType(child) & SpecObject),
6894 unsure(functionCase), unsure(notFunctionCase));
6895
6896 m_out.appendTo(functionCase, notFunctionCase);
6897 functor(TypeofType::Function);
6898
6899 m_out.appendTo(notFunctionCase, reallyObjectCase);
6900 m_out.branch(
6901 isExoticForTypeof(value, provenType(child) & (SpecObject - SpecFunction)),
6902 rarely(slowPath), usually(reallyObjectCase));
6903
6904 m_out.appendTo(reallyObjectCase, slowPath);
6905 functor(TypeofType::Object);
6906
6907 m_out.appendTo(slowPath, unreachable);
6908 LValue result = vmCall(
6909 m_out.operation(operationTypeOfObjectAsTypeofType), m_callFrame,
6910 weakPointer(globalObject), value);
6911 Vector<SwitchCase, 3> cases;
6912 cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Undefined)), undefinedCase));
6913 cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Object)), reallyObjectCase));
6914 cases.append(SwitchCase(m_out.constInt32(static_cast<int32_t>(TypeofType::Function)), functionCase));
6915 m_out.switchInstruction(result, cases, unreachable, Weight());
6916
6917 m_out.appendTo(unreachable, notObjectCase);
6918 m_out.unreachable();
6919
6920 m_out.appendTo(notObjectCase, stringCase);
6921 m_out.branch(
6922 isString(value, provenType(child) & (SpecCell - SpecObject)),
6923 unsure(stringCase), unsure(symbolCase));
6924
6925 m_out.appendTo(stringCase, symbolCase);
6926 functor(TypeofType::String);
6927
6928 m_out.appendTo(symbolCase, notCellCase);
6929 functor(TypeofType::Symbol);
6930
6931 m_out.appendTo(notCellCase, numberCase);
6932 m_out.branch(
6933 isNumber(value, provenType(child) & ~SpecCell),
6934 unsure(numberCase), unsure(notNumberCase));
6935
6936 m_out.appendTo(numberCase, notNumberCase);
6937 functor(TypeofType::Number);
6938
6939 m_out.appendTo(notNumberCase, notNullCase);
6940 LValue isNull;
6941 if (provenType(child) & SpecOther)
6942 isNull = m_out.equal(value, m_out.constInt64(ValueNull));
6943 else
6944 isNull = m_out.booleanFalse;
6945 m_out.branch(isNull, unsure(reallyObjectCase), unsure(notNullCase));
6946
6947 m_out.appendTo(notNullCase, booleanCase);
6948 m_out.branch(
6949 isBoolean(value, provenType(child) & ~(SpecCell | SpecFullNumber)),
6950 unsure(booleanCase), unsure(undefinedCase));
6951
6952 m_out.appendTo(booleanCase, undefinedCase);
6953 functor(TypeofType::Boolean);
6954
6955 m_out.appendTo(undefinedCase, lastNext);
6956 functor(TypeofType::Undefined);
6957 }
6958
6959 LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true)
6960 {
6961 LBasicBlock greatEnough = FTL_NEW_BLOCK(m_out, ("doubleToInt32 greatEnough"));
6962 LBasicBlock withinRange = FTL_NEW_BLOCK(m_out, ("doubleToInt32 withinRange"));
6963 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("doubleToInt32 slowPath"));
6964 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("doubleToInt32 continuation"));
6965
6966 Vector<ValueFromBlock, 2> results;
6967
6968 m_out.branch(
6969 m_out.doubleGreaterThanOrEqual(doubleValue, m_out.constDouble(low)),
6970 unsure(greatEnough), unsure(slowPath));
6971
6972 LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange);
6973 m_out.branch(
6974 m_out.doubleLessThanOrEqual(doubleValue, m_out.constDouble(high)),
6975 unsure(withinRange), unsure(slowPath));
6976
6977 m_out.appendTo(withinRange, slowPath);
6978 LValue fastResult;
6979 if (isSigned)
6980 fastResult = m_out.fpToInt32(doubleValue);
6981 else
6982 fastResult = m_out.fpToUInt32(doubleValue);
6983 results.append(m_out.anchor(fastResult));
6984 m_out.jump(continuation);
6985
6986 m_out.appendTo(slowPath, continuation);
6987 results.append(m_out.anchor(m_out.call(m_out.operation(toInt32), doubleValue)));
6988 m_out.jump(continuation);
6989
6990 m_out.appendTo(continuation, lastNext);
6991 return m_out.phi(m_out.int32, results);
6992 }
6993
6994 LValue doubleToInt32(LValue doubleValue)
6995 {
6996 if (Output::hasSensibleDoubleToInt())
6997 return sensibleDoubleToInt32(doubleValue);
6998
6999 double limit = pow(2, 31) - 1;
7000 return doubleToInt32(doubleValue, -limit, limit);
7001 }
7002
7003 LValue sensibleDoubleToInt32(LValue doubleValue)
7004 {
7005 LBasicBlock slowPath = FTL_NEW_BLOCK(m_out, ("sensible doubleToInt32 slow path"));
7006 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("sensible doubleToInt32 continuation"));
7007
7008 ValueFromBlock fastResult = m_out.anchor(
7009 m_out.sensibleDoubleToInt(doubleValue));
7010 m_out.branch(
7011 m_out.equal(fastResult.value(), m_out.constInt32(0x80000000)),
7012 rarely(slowPath), usually(continuation));
7013
7014 LBasicBlock lastNext = m_out.appendTo(slowPath, continuation);
7015 ValueFromBlock slowResult = m_out.anchor(
7016 m_out.call(m_out.operation(toInt32), doubleValue));
7017 m_out.jump(continuation);
7018
7019 m_out.appendTo(continuation, lastNext);
7020 return m_out.phi(m_out.int32, fastResult, slowResult);
7021 }
7022
7023 void speculate(
7024 ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
7025 {
7026 appendOSRExit(kind, lowValue, highValue, failCondition);
7027 }
7028
7029 void terminate(ExitKind kind)
7030 {
7031 speculate(kind, noValue(), nullptr, m_out.booleanTrue);
7032 didAlreadyTerminate();
7033 }
7034
7035 void didAlreadyTerminate()
7036 {
7037 m_state.setIsValid(false);
7038 }
7039
7040 void typeCheck(
7041 FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
7042 LValue failCondition)
7043 {
7044 appendTypeCheck(lowValue, highValue, typesPassedThrough, failCondition);
7045 }
7046
7047 void appendTypeCheck(
7048 FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough,
7049 LValue failCondition)
7050 {
7051 if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough))
7052 return;
7053 ASSERT(mayHaveTypeCheck(highValue.useKind()));
7054 appendOSRExit(BadType, lowValue, highValue.node(), failCondition);
7055 m_interpreter.filter(highValue, typesPassedThrough);
7056 }
7057
7058 LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7059 {
7060 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use));
7061
7062 if (edge->hasConstant()) {
7063 JSValue value = edge->asJSValue();
7064 if (!value.isInt32()) {
7065 terminate(Uncountable);
7066 return m_out.int32Zero;
7067 }
7068 return m_out.constInt32(value.asInt32());
7069 }
7070
7071 LoweredNodeValue value = m_int32Values.get(edge.node());
7072 if (isValid(value))
7073 return value.value();
7074
7075 value = m_strictInt52Values.get(edge.node());
7076 if (isValid(value))
7077 return strictInt52ToInt32(edge, value.value());
7078
7079 value = m_int52Values.get(edge.node());
7080 if (isValid(value))
7081 return strictInt52ToInt32(edge, int52ToStrictInt52(value.value()));
7082
7083 value = m_jsValueValues.get(edge.node());
7084 if (isValid(value)) {
7085 LValue boxedResult = value.value();
7086 FTL_TYPE_CHECK(
7087 jsValueValue(boxedResult), edge, SpecInt32, isNotInt32(boxedResult));
7088 LValue result = unboxInt32(boxedResult);
7089 setInt32(edge.node(), result);
7090 return result;
7091 }
7092
7093 DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecInt32));
7094 terminate(Uncountable);
7095 return m_out.int32Zero;
7096 }
7097
7098 enum Int52Kind { StrictInt52, Int52 };
7099 LValue lowInt52(Edge edge, Int52Kind kind)
7100 {
7101 DFG_ASSERT(m_graph, m_node, edge.useKind() == Int52RepUse);
7102
7103 LoweredNodeValue value;
7104
7105 switch (kind) {
7106 case Int52:
7107 value = m_int52Values.get(edge.node());
7108 if (isValid(value))
7109 return value.value();
7110
7111 value = m_strictInt52Values.get(edge.node());
7112 if (isValid(value))
7113 return strictInt52ToInt52(value.value());
7114 break;
7115
7116 case StrictInt52:
7117 value = m_strictInt52Values.get(edge.node());
7118 if (isValid(value))
7119 return value.value();
7120
7121 value = m_int52Values.get(edge.node());
7122 if (isValid(value))
7123 return int52ToStrictInt52(value.value());
7124 break;
7125 }
7126
7127 DFG_ASSERT(m_graph, m_node, !provenType(edge));
7128 terminate(Uncountable);
7129 return m_out.int64Zero;
7130 }
7131
7132 LValue lowInt52(Edge edge)
7133 {
7134 return lowInt52(edge, Int52);
7135 }
7136
7137 LValue lowStrictInt52(Edge edge)
7138 {
7139 return lowInt52(edge, StrictInt52);
7140 }
7141
7142 bool betterUseStrictInt52(Node* node)
7143 {
7144 return !isValid(m_int52Values.get(node));
7145 }
7146 bool betterUseStrictInt52(Edge edge)
7147 {
7148 return betterUseStrictInt52(edge.node());
7149 }
7150 template<typename T>
7151 Int52Kind bestInt52Kind(T node)
7152 {
7153 return betterUseStrictInt52(node) ? StrictInt52 : Int52;
7154 }
7155 Int52Kind opposite(Int52Kind kind)
7156 {
7157 switch (kind) {
7158 case Int52:
7159 return StrictInt52;
7160 case StrictInt52:
7161 return Int52;
7162 }
7163 DFG_CRASH(m_graph, m_node, "Bad use kind");
7164 return Int52;
7165 }
7166
7167 LValue lowWhicheverInt52(Edge edge, Int52Kind& kind)
7168 {
7169 kind = bestInt52Kind(edge);
7170 return lowInt52(edge, kind);
7171 }
7172
7173 LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7174 {
7175 DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || DFG::isCell(edge.useKind()));
7176
7177 if (edge->op() == JSConstant) {
7178 JSValue value = edge->asJSValue();
7179 if (!value.isCell()) {
7180 terminate(Uncountable);
7181 return m_out.intPtrZero;
7182 }
7183 return m_out.constIntPtr(value.asCell());
7184 }
7185
7186 LoweredNodeValue value = m_jsValueValues.get(edge.node());
7187 if (isValid(value)) {
7188 LValue uncheckedValue = value.value();
7189 FTL_TYPE_CHECK(
7190 jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue));
7191 return uncheckedValue;
7192 }
7193
7194 DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecCell));
7195 terminate(Uncountable);
7196 return m_out.intPtrZero;
7197 }
7198
7199 LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7200 {
7201 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
7202
7203 LValue result = lowCell(edge, mode);
7204 speculateObject(edge, result);
7205 return result;
7206 }
7207
7208 LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7209 {
7210 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse || edge.useKind() == StringIdentUse);
7211
7212 LValue result = lowCell(edge, mode);
7213 speculateString(edge, result);
7214 return result;
7215 }
7216
7217 LValue lowStringIdent(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7218 {
7219 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringIdentUse);
7220
7221 LValue string = lowString(edge, mode);
7222 LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value);
7223 speculateStringIdent(edge, string, stringImpl);
7224 return stringImpl;
7225 }
7226
7227 LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7228 {
7229 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse);
7230
7231 LValue result = lowCell(edge, mode);
7232 speculateNonNullObject(edge, result);
7233 return result;
7234 }
7235
7236 LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7237 {
7238 ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse);
7239
7240 if (edge->hasConstant()) {
7241 JSValue value = edge->asJSValue();
7242 if (!value.isBoolean()) {
7243 terminate(Uncountable);
7244 return m_out.booleanFalse;
7245 }
7246 return m_out.constBool(value.asBoolean());
7247 }
7248
7249 LoweredNodeValue value = m_booleanValues.get(edge.node());
7250 if (isValid(value))
7251 return value.value();
7252
7253 value = m_jsValueValues.get(edge.node());
7254 if (isValid(value)) {
7255 LValue unboxedResult = value.value();
7256 FTL_TYPE_CHECK(
7257 jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult));
7258 LValue result = unboxBoolean(unboxedResult);
7259 setBoolean(edge.node(), result);
7260 return result;
7261 }
7262
7263 DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecBoolean));
7264 terminate(Uncountable);
7265 return m_out.booleanFalse;
7266 }
7267
7268 LValue lowDouble(Edge edge)
7269 {
7270 DFG_ASSERT(m_graph, m_node, isDouble(edge.useKind()));
7271
7272 LoweredNodeValue value = m_doubleValues.get(edge.node());
7273 if (isValid(value))
7274 return value.value();
7275 DFG_ASSERT(m_graph, m_node, !provenType(edge));
7276 terminate(Uncountable);
7277 return m_out.doubleZero;
7278 }
7279
7280 LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation)
7281 {
7282 DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse);
7283 DFG_ASSERT(m_graph, m_node, !isDouble(edge.useKind()));
7284 DFG_ASSERT(m_graph, m_node, edge.useKind() != Int52RepUse);
7285
7286 if (edge->hasConstant())
7287 return m_out.constInt64(JSValue::encode(edge->asJSValue()));
7288
7289 LoweredNodeValue value = m_jsValueValues.get(edge.node());
7290 if (isValid(value))
7291 return value.value();
7292
7293 value = m_int32Values.get(edge.node());
7294 if (isValid(value)) {
7295 LValue result = boxInt32(value.value());
7296 setJSValue(edge.node(), result);
7297 return result;
7298 }
7299
7300 value = m_booleanValues.get(edge.node());
7301 if (isValid(value)) {
7302 LValue result = boxBoolean(value.value());
7303 setJSValue(edge.node(), result);
7304 return result;
7305 }
7306
7307 DFG_CRASH(m_graph, m_node, "Value not defined");
7308 return 0;
7309 }
7310
7311 LValue lowStorage(Edge edge)
7312 {
7313 LoweredNodeValue value = m_storageValues.get(edge.node());
7314 if (isValid(value))
7315 return value.value();
7316
7317 LValue result = lowCell(edge);
7318 setStorage(edge.node(), result);
7319 return result;
7320 }
7321
7322 LValue strictInt52ToInt32(Edge edge, LValue value)
7323 {
7324 LValue result = m_out.castToInt32(value);
7325 FTL_TYPE_CHECK(
7326 noValue(), edge, SpecInt32,
7327 m_out.notEqual(m_out.signExt(result, m_out.int64), value));
7328 setInt32(edge.node(), result);
7329 return result;
7330 }
7331
7332 LValue strictInt52ToDouble(LValue value)
7333 {
7334 return m_out.intToDouble(value);
7335 }
7336
7337 LValue strictInt52ToJSValue(LValue value)
7338 {
7339 LBasicBlock isInt32 = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isInt32 case"));
7340 LBasicBlock isDouble = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue isDouble case"));
7341 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("strictInt52ToJSValue continuation"));
7342
7343 Vector<ValueFromBlock, 2> results;
7344
7345 LValue int32Value = m_out.castToInt32(value);
7346 m_out.branch(
7347 m_out.equal(m_out.signExt(int32Value, m_out.int64), value),
7348 unsure(isInt32), unsure(isDouble));
7349
7350 LBasicBlock lastNext = m_out.appendTo(isInt32, isDouble);
7351
7352 results.append(m_out.anchor(boxInt32(int32Value)));
7353 m_out.jump(continuation);
7354
7355 m_out.appendTo(isDouble, continuation);
7356
7357 results.append(m_out.anchor(boxDouble(m_out.intToDouble(value))));
7358 m_out.jump(continuation);
7359
7360 m_out.appendTo(continuation, lastNext);
7361 return m_out.phi(m_out.int64, results);
7362 }
7363
7364 LValue strictInt52ToInt52(LValue value)
7365 {
7366 return m_out.shl(value, m_out.constInt64(JSValue::int52ShiftAmount));
7367 }
7368
7369 LValue int52ToStrictInt52(LValue value)
7370 {
7371 return m_out.aShr(value, m_out.constInt64(JSValue::int52ShiftAmount));
7372 }
7373
7374 LValue isInt32(LValue jsValue, SpeculatedType type = SpecFullTop)
7375 {
7376 if (LValue proven = isProvenValue(type, SpecInt32))
7377 return proven;
7378 return m_out.aboveOrEqual(jsValue, m_tagTypeNumber);
7379 }
7380 LValue isNotInt32(LValue jsValue, SpeculatedType type = SpecFullTop)
7381 {
7382 if (LValue proven = isProvenValue(type, ~SpecInt32))
7383 return proven;
7384 return m_out.below(jsValue, m_tagTypeNumber);
7385 }
7386 LValue unboxInt32(LValue jsValue)
7387 {
7388 return m_out.castToInt32(jsValue);
7389 }
7390 LValue boxInt32(LValue value)
7391 {
7392 return m_out.add(m_out.zeroExt(value, m_out.int64), m_tagTypeNumber);
7393 }
7394
7395 LValue isCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop)
7396 {
7397 if (LValue proven = isProvenValue(type, SpecCell | SpecMisc))
7398 return proven;
7399 return m_out.testIsZero64(jsValue, m_tagTypeNumber);
7400 }
7401 LValue isNotCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop)
7402 {
7403 if (LValue proven = isProvenValue(type, ~(SpecCell | SpecMisc)))
7404 return proven;
7405 return m_out.testNonZero64(jsValue, m_tagTypeNumber);
7406 }
7407
7408 LValue unboxDouble(LValue jsValue)
7409 {
7410 return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), m_out.doubleType);
7411 }
7412 LValue boxDouble(LValue doubleValue)
7413 {
7414 return m_out.sub(m_out.bitCast(doubleValue, m_out.int64), m_tagTypeNumber);
7415 }
7416
7417 LValue jsValueToStrictInt52(Edge edge, LValue boxedValue)
7418 {
7419 LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing int case"));
7420 LBasicBlock doubleCase = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing double case"));
7421 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("jsValueToInt52 unboxing continuation"));
7422
7423 LValue isNotInt32;
7424 if (!m_interpreter.needsTypeCheck(edge, SpecInt32))
7425 isNotInt32 = m_out.booleanFalse;
7426 else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32))
7427 isNotInt32 = m_out.booleanTrue;
7428 else
7429 isNotInt32 = this->isNotInt32(boxedValue);
7430 m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase));
7431
7432 LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase);
7433
7434 ValueFromBlock intToInt52 = m_out.anchor(
7435 m_out.signExt(unboxInt32(boxedValue), m_out.int64));
7436 m_out.jump(continuation);
7437
7438 m_out.appendTo(doubleCase, continuation);
7439
7440 LValue possibleResult = m_out.call(
7441 m_out.operation(operationConvertBoxedDoubleToInt52), boxedValue);
7442 FTL_TYPE_CHECK(
7443 jsValueValue(boxedValue), edge, SpecInt32 | SpecInt52AsDouble,
7444 m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
7445
7446 ValueFromBlock doubleToInt52 = m_out.anchor(possibleResult);
7447 m_out.jump(continuation);
7448
7449 m_out.appendTo(continuation, lastNext);
7450
7451 return m_out.phi(m_out.int64, intToInt52, doubleToInt52);
7452 }
7453
7454 LValue doubleToStrictInt52(Edge edge, LValue value)
7455 {
7456 LValue possibleResult = m_out.call(
7457 m_out.operation(operationConvertDoubleToInt52), value);
7458 FTL_TYPE_CHECK(
7459 doubleValue(value), edge, SpecInt52AsDouble,
7460 m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52)));
7461
7462 return possibleResult;
7463 }
7464
7465 LValue convertDoubleToInt32(LValue value, bool shouldCheckNegativeZero)
7466 {
7467 LValue integerValue = m_out.fpToInt32(value);
7468 LValue integerValueConvertedToDouble = m_out.intToDouble(integerValue);
7469 LValue valueNotConvertibleToInteger = m_out.doubleNotEqualOrUnordered(value, integerValueConvertedToDouble);
7470 speculate(Overflow, FormattedValue(ValueFormatDouble, value), m_node, valueNotConvertibleToInteger);
7471
7472 if (shouldCheckNegativeZero) {
7473 LBasicBlock valueIsZero = FTL_NEW_BLOCK(m_out, ("ConvertDoubleToInt32 on zero"));
7474 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("ConvertDoubleToInt32 continuation"));
7475 m_out.branch(m_out.isZero32(integerValue), unsure(valueIsZero), unsure(continuation));
7476
7477 LBasicBlock lastNext = m_out.appendTo(valueIsZero, continuation);
7478
7479 LValue doubleBitcastToInt64 = m_out.bitCast(value, m_out.int64);
7480 LValue signBitSet = m_out.lessThan(doubleBitcastToInt64, m_out.constInt64(0));
7481
7482 speculate(NegativeZero, FormattedValue(ValueFormatDouble, value), m_node, signBitSet);
7483 m_out.jump(continuation);
7484 m_out.appendTo(continuation, lastNext);
7485 }
7486 return integerValue;
7487 }
7488
7489 LValue isNumber(LValue jsValue, SpeculatedType type = SpecFullTop)
7490 {
7491 if (LValue proven = isProvenValue(type, SpecFullNumber))
7492 return proven;
7493 return isNotCellOrMisc(jsValue);
7494 }
7495 LValue isNotNumber(LValue jsValue, SpeculatedType type = SpecFullTop)
7496 {
7497 if (LValue proven = isProvenValue(type, ~SpecFullNumber))
7498 return proven;
7499 return isCellOrMisc(jsValue);
7500 }
7501
7502 LValue isNotCell(LValue jsValue, SpeculatedType type = SpecFullTop)
7503 {
7504 if (LValue proven = isProvenValue(type, ~SpecCell))
7505 return proven;
7506 return m_out.testNonZero64(jsValue, m_tagMask);
7507 }
7508
7509 LValue isCell(LValue jsValue, SpeculatedType type = SpecFullTop)
7510 {
7511 if (LValue proven = isProvenValue(type, SpecCell))
7512 return proven;
7513 return m_out.testIsZero64(jsValue, m_tagMask);
7514 }
7515
7516 LValue isNotMisc(LValue value, SpeculatedType type = SpecFullTop)
7517 {
7518 if (LValue proven = isProvenValue(type, ~SpecMisc))
7519 return proven;
7520 return m_out.above(value, m_out.constInt64(TagBitTypeOther | TagBitBool | TagBitUndefined));
7521 }
7522
7523 LValue isMisc(LValue value, SpeculatedType type = SpecFullTop)
7524 {
7525 if (LValue proven = isProvenValue(type, SpecMisc))
7526 return proven;
7527 return m_out.bitNot(isNotMisc(value));
7528 }
7529
7530 LValue isNotBoolean(LValue jsValue, SpeculatedType type = SpecFullTop)
7531 {
7532 if (LValue proven = isProvenValue(type, ~SpecBoolean))
7533 return proven;
7534 return m_out.testNonZero64(
7535 m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)),
7536 m_out.constInt64(~1));
7537 }
7538 LValue isBoolean(LValue jsValue, SpeculatedType type = SpecFullTop)
7539 {
7540 if (LValue proven = isProvenValue(type, SpecBoolean))
7541 return proven;
7542 return m_out.bitNot(isNotBoolean(jsValue));
7543 }
7544 LValue unboxBoolean(LValue jsValue)
7545 {
7546 // We want to use a cast that guarantees that LLVM knows that even the integer
7547 // value is just 0 or 1. But for now we do it the dumb way.
7548 return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1)));
7549 }
7550 LValue boxBoolean(LValue value)
7551 {
7552 return m_out.select(
7553 value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse));
7554 }
7555
7556 LValue isNotOther(LValue value, SpeculatedType type = SpecFullTop)
7557 {
7558 if (LValue proven = isProvenValue(type, ~SpecOther))
7559 return proven;
7560 return m_out.notEqual(
7561 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
7562 m_out.constInt64(ValueNull));
7563 }
7564 LValue isOther(LValue value, SpeculatedType type = SpecFullTop)
7565 {
7566 if (LValue proven = isProvenValue(type, SpecOther))
7567 return proven;
7568 return m_out.equal(
7569 m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)),
7570 m_out.constInt64(ValueNull));
7571 }
7572
7573 LValue isProvenValue(SpeculatedType provenType, SpeculatedType wantedType)
7574 {
7575 if (!(provenType & ~wantedType))
7576 return m_out.booleanTrue;
7577 if (!(provenType & wantedType))
7578 return m_out.booleanFalse;
7579 return nullptr;
7580 }
7581
7582 void speculate(Edge edge)
7583 {
7584 switch (edge.useKind()) {
7585 case UntypedUse:
7586 break;
7587 case KnownInt32Use:
7588 case KnownStringUse:
7589 case DoubleRepUse:
7590 case Int52RepUse:
7591 ASSERT(!m_interpreter.needsTypeCheck(edge));
7592 break;
7593 case Int32Use:
7594 speculateInt32(edge);
7595 break;
7596 case CellUse:
7597 speculateCell(edge);
7598 break;
7599 case KnownCellUse:
7600 ASSERT(!m_interpreter.needsTypeCheck(edge));
7601 break;
7602 case MachineIntUse:
7603 speculateMachineInt(edge);
7604 break;
7605 case ObjectUse:
7606 speculateObject(edge);
7607 break;
7608 case FunctionUse:
7609 speculateFunction(edge);
7610 break;
7611 case ObjectOrOtherUse:
7612 speculateObjectOrOther(edge);
7613 break;
7614 case FinalObjectUse:
7615 speculateFinalObject(edge);
7616 break;
7617 case StringUse:
7618 speculateString(edge);
7619 break;
7620 case StringIdentUse:
7621 speculateStringIdent(edge);
7622 break;
7623 case StringObjectUse:
7624 speculateStringObject(edge);
7625 break;
7626 case StringOrStringObjectUse:
7627 speculateStringOrStringObject(edge);
7628 break;
7629 case NumberUse:
7630 speculateNumber(edge);
7631 break;
7632 case RealNumberUse:
7633 speculateRealNumber(edge);
7634 break;
7635 case DoubleRepRealUse:
7636 speculateDoubleRepReal(edge);
7637 break;
7638 case DoubleRepMachineIntUse:
7639 speculateDoubleRepMachineInt(edge);
7640 break;
7641 case BooleanUse:
7642 speculateBoolean(edge);
7643 break;
7644 case NotStringVarUse:
7645 speculateNotStringVar(edge);
7646 break;
7647 case NotCellUse:
7648 speculateNotCell(edge);
7649 break;
7650 case OtherUse:
7651 speculateOther(edge);
7652 break;
7653 case MiscUse:
7654 speculateMisc(edge);
7655 break;
7656 default:
7657 DFG_CRASH(m_graph, m_node, "Unsupported speculation use kind");
7658 }
7659 }
7660
7661 void speculate(Node*, Edge edge)
7662 {
7663 speculate(edge);
7664 }
7665
7666 void speculateInt32(Edge edge)
7667 {
7668 lowInt32(edge);
7669 }
7670
7671 void speculateCell(Edge edge)
7672 {
7673 lowCell(edge);
7674 }
7675
7676 void speculateMachineInt(Edge edge)
7677 {
7678 if (!m_interpreter.needsTypeCheck(edge))
7679 return;
7680
7681 jsValueToStrictInt52(edge, lowJSValue(edge, ManualOperandSpeculation));
7682 }
7683
7684 LValue isObject(LValue cell, SpeculatedType type = SpecFullTop)
7685 {
7686 if (LValue proven = isProvenValue(type & SpecCell, SpecObject))
7687 return proven;
7688 return m_out.aboveOrEqual(
7689 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7690 m_out.constInt8(ObjectType));
7691 }
7692
7693 LValue isNotObject(LValue cell, SpeculatedType type = SpecFullTop)
7694 {
7695 if (LValue proven = isProvenValue(type & SpecCell, ~SpecObject))
7696 return proven;
7697 return m_out.below(
7698 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7699 m_out.constInt8(ObjectType));
7700 }
7701
7702 LValue isNotString(LValue cell, SpeculatedType type = SpecFullTop)
7703 {
7704 if (LValue proven = isProvenValue(type & SpecCell, ~SpecString))
7705 return proven;
7706 return m_out.notEqual(
7707 m_out.load32(cell, m_heaps.JSCell_structureID),
7708 m_out.constInt32(vm().stringStructure->id()));
7709 }
7710
7711 LValue isString(LValue cell, SpeculatedType type = SpecFullTop)
7712 {
7713 if (LValue proven = isProvenValue(type & SpecCell, SpecString))
7714 return proven;
7715 return m_out.equal(
7716 m_out.load32(cell, m_heaps.JSCell_structureID),
7717 m_out.constInt32(vm().stringStructure->id()));
7718 }
7719
7720 LValue isArrayType(LValue cell, ArrayMode arrayMode)
7721 {
7722 switch (arrayMode.type()) {
7723 case Array::Int32:
7724 case Array::Double:
7725 case Array::Contiguous: {
7726 LValue indexingType = m_out.load8(cell, m_heaps.JSCell_indexingType);
7727
7728 switch (arrayMode.arrayClass()) {
7729 case Array::OriginalArray:
7730 DFG_CRASH(m_graph, m_node, "Unexpected original array");
7731 return 0;
7732
7733 case Array::Array:
7734 return m_out.equal(
7735 m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
7736 m_out.constInt8(IsArray | arrayMode.shapeMask()));
7737
7738 case Array::NonArray:
7739 case Array::OriginalNonArray:
7740 return m_out.equal(
7741 m_out.bitAnd(indexingType, m_out.constInt8(IsArray | IndexingShapeMask)),
7742 m_out.constInt8(arrayMode.shapeMask()));
7743
7744 case Array::PossiblyArray:
7745 return m_out.equal(
7746 m_out.bitAnd(indexingType, m_out.constInt8(IndexingShapeMask)),
7747 m_out.constInt8(arrayMode.shapeMask()));
7748 }
7749
7750 DFG_CRASH(m_graph, m_node, "Corrupt array class");
7751 }
7752
7753 case Array::DirectArguments:
7754 return m_out.equal(
7755 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7756 m_out.constInt8(DirectArgumentsType));
7757
7758 case Array::ScopedArguments:
7759 return m_out.equal(
7760 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7761 m_out.constInt8(ScopedArgumentsType));
7762
7763 default:
7764 return m_out.equal(
7765 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7766 m_out.constInt8(typeForTypedArrayType(arrayMode.typedArrayType())));
7767 }
7768 }
7769
7770 LValue isFunction(LValue cell, SpeculatedType type = SpecFullTop)
7771 {
7772 if (LValue proven = isProvenValue(type & SpecCell, SpecFunction))
7773 return proven;
7774 return isType(cell, JSFunctionType);
7775 }
7776 LValue isNotFunction(LValue cell, SpeculatedType type = SpecFullTop)
7777 {
7778 if (LValue proven = isProvenValue(type & SpecCell, ~SpecFunction))
7779 return proven;
7780 return isNotType(cell, JSFunctionType);
7781 }
7782
7783 LValue isExoticForTypeof(LValue cell, SpeculatedType type = SpecFullTop)
7784 {
7785 if (!(type & SpecObjectOther))
7786 return m_out.booleanFalse;
7787 return m_out.testNonZero8(
7788 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
7789 m_out.constInt8(MasqueradesAsUndefined | TypeOfShouldCallGetCallData));
7790 }
7791
7792 LValue isType(LValue cell, JSType type)
7793 {
7794 return m_out.equal(
7795 m_out.load8(cell, m_heaps.JSCell_typeInfoType),
7796 m_out.constInt8(type));
7797 }
7798
7799 LValue isNotType(LValue cell, JSType type)
7800 {
7801 return m_out.bitNot(isType(cell, type));
7802 }
7803
7804 void speculateObject(Edge edge, LValue cell)
7805 {
7806 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
7807 }
7808
7809 void speculateObject(Edge edge)
7810 {
7811 speculateObject(edge, lowCell(edge));
7812 }
7813
7814 void speculateFunction(Edge edge, LValue cell)
7815 {
7816 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecFunction, isNotFunction(cell));
7817 }
7818
7819 void speculateFunction(Edge edge)
7820 {
7821 speculateFunction(edge, lowCell(edge));
7822 }
7823
7824 void speculateObjectOrOther(Edge edge)
7825 {
7826 if (!m_interpreter.needsTypeCheck(edge))
7827 return;
7828
7829 LValue value = lowJSValue(edge, ManualOperandSpeculation);
7830
7831 LBasicBlock cellCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther cell case"));
7832 LBasicBlock primitiveCase = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther primitive case"));
7833 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateObjectOrOther continuation"));
7834
7835 m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase));
7836
7837 LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase);
7838
7839 FTL_TYPE_CHECK(
7840 jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value));
7841
7842 m_out.jump(continuation);
7843
7844 m_out.appendTo(primitiveCase, continuation);
7845
7846 FTL_TYPE_CHECK(
7847 jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value));
7848
7849 m_out.jump(continuation);
7850
7851 m_out.appendTo(continuation, lastNext);
7852 }
7853
7854 void speculateFinalObject(Edge edge, LValue cell)
7855 {
7856 FTL_TYPE_CHECK(
7857 jsValueValue(cell), edge, SpecFinalObject, isNotType(cell, FinalObjectType));
7858 }
7859
7860 void speculateFinalObject(Edge edge)
7861 {
7862 speculateFinalObject(edge, lowCell(edge));
7863 }
7864
7865 void speculateString(Edge edge, LValue cell)
7866 {
7867 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString | ~SpecCell, isNotString(cell));
7868 }
7869
7870 void speculateString(Edge edge)
7871 {
7872 speculateString(edge, lowCell(edge));
7873 }
7874
7875 void speculateStringIdent(Edge edge, LValue string, LValue stringImpl)
7876 {
7877 if (!m_interpreter.needsTypeCheck(edge, SpecStringIdent | ~SpecString))
7878 return;
7879
7880 speculate(BadType, jsValueValue(string), edge.node(), m_out.isNull(stringImpl));
7881 speculate(
7882 BadType, jsValueValue(string), edge.node(),
7883 m_out.testIsZero32(
7884 m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags),
7885 m_out.constInt32(StringImpl::flagIsAtomic())));
7886 m_interpreter.filter(edge, SpecStringIdent | ~SpecString);
7887 }
7888
7889 void speculateStringIdent(Edge edge)
7890 {
7891 lowStringIdent(edge);
7892 }
7893
7894 void speculateStringObject(Edge edge)
7895 {
7896 if (!m_interpreter.needsTypeCheck(edge, SpecStringObject))
7897 return;
7898
7899 speculateStringObjectForCell(edge, lowCell(edge));
7900 m_interpreter.filter(edge, SpecStringObject);
7901 }
7902
7903 void speculateStringOrStringObject(Edge edge)
7904 {
7905 if (!m_interpreter.needsTypeCheck(edge, SpecString | SpecStringObject))
7906 return;
7907
7908 LBasicBlock notString = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject not string case"));
7909 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate StringOrStringObject continuation"));
7910
7911 LValue structureID = m_out.load32(lowCell(edge), m_heaps.JSCell_structureID);
7912 m_out.branch(
7913 m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())),
7914 unsure(continuation), unsure(notString));
7915
7916 LBasicBlock lastNext = m_out.appendTo(notString, continuation);
7917 speculateStringObjectForStructureID(edge, structureID);
7918 m_out.jump(continuation);
7919
7920 m_out.appendTo(continuation, lastNext);
7921
7922 m_interpreter.filter(edge, SpecString | SpecStringObject);
7923 }
7924
7925 void speculateStringObjectForCell(Edge edge, LValue cell)
7926 {
7927 speculateStringObjectForStructureID(edge, m_out.load32(cell, m_heaps.JSCell_structureID));
7928 }
7929
7930 void speculateStringObjectForStructureID(Edge edge, LValue structureID)
7931 {
7932 Structure* stringObjectStructure =
7933 m_graph.globalObjectFor(m_node->origin.semantic)->stringObjectStructure();
7934
7935 if (abstractStructure(edge).isSubsetOf(StructureSet(stringObjectStructure)))
7936 return;
7937
7938 speculate(
7939 NotStringObject, noValue(), 0,
7940 m_out.notEqual(structureID, weakStructureID(stringObjectStructure)));
7941 }
7942
7943 void speculateNonNullObject(Edge edge, LValue cell)
7944 {
7945 FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell));
7946 if (masqueradesAsUndefinedWatchpointIsStillValid())
7947 return;
7948
7949 speculate(
7950 BadType, jsValueValue(cell), edge.node(),
7951 m_out.testNonZero8(
7952 m_out.load8(cell, m_heaps.JSCell_typeInfoFlags),
7953 m_out.constInt8(MasqueradesAsUndefined)));
7954 }
7955
7956 void speculateNumber(Edge edge)
7957 {
7958 LValue value = lowJSValue(edge, ManualOperandSpeculation);
7959 FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isNotNumber(value));
7960 }
7961
7962 void speculateRealNumber(Edge edge)
7963 {
7964 // Do an early return here because lowDouble() can create a lot of control flow.
7965 if (!m_interpreter.needsTypeCheck(edge))
7966 return;
7967
7968 LValue value = lowJSValue(edge, ManualOperandSpeculation);
7969 LValue doubleValue = unboxDouble(value);
7970
7971 LBasicBlock intCase = FTL_NEW_BLOCK(m_out, ("speculateRealNumber int case"));
7972 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("speculateRealNumber continuation"));
7973
7974 m_out.branch(
7975 m_out.doubleEqual(doubleValue, doubleValue),
7976 usually(continuation), rarely(intCase));
7977
7978 LBasicBlock lastNext = m_out.appendTo(intCase, continuation);
7979
7980 typeCheck(
7981 jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber,
7982 isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble));
7983 m_out.jump(continuation);
7984
7985 m_out.appendTo(continuation, lastNext);
7986 }
7987
7988 void speculateDoubleRepReal(Edge edge)
7989 {
7990 // Do an early return here because lowDouble() can create a lot of control flow.
7991 if (!m_interpreter.needsTypeCheck(edge))
7992 return;
7993
7994 LValue value = lowDouble(edge);
7995 FTL_TYPE_CHECK(
7996 doubleValue(value), edge, SpecDoubleReal,
7997 m_out.doubleNotEqualOrUnordered(value, value));
7998 }
7999
8000 void speculateDoubleRepMachineInt(Edge edge)
8001 {
8002 if (!m_interpreter.needsTypeCheck(edge))
8003 return;
8004
8005 doubleToStrictInt52(edge, lowDouble(edge));
8006 }
8007
8008 void speculateBoolean(Edge edge)
8009 {
8010 lowBoolean(edge);
8011 }
8012
8013 void speculateNotStringVar(Edge edge)
8014 {
8015 if (!m_interpreter.needsTypeCheck(edge, ~SpecStringVar))
8016 return;
8017
8018 LValue value = lowJSValue(edge, ManualOperandSpeculation);
8019
8020 LBasicBlock isCellCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is cell case"));
8021 LBasicBlock isStringCase = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar is string case"));
8022 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Speculate NotStringVar continuation"));
8023
8024 m_out.branch(isCell(value, provenType(edge)), unsure(isCellCase), unsure(continuation));
8025
8026 LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase);
8027 m_out.branch(isString(value, provenType(edge)), unsure(isStringCase), unsure(continuation));
8028
8029 m_out.appendTo(isStringCase, continuation);
8030 speculateStringIdent(edge, value, m_out.loadPtr(value, m_heaps.JSString_value));
8031 m_out.jump(continuation);
8032
8033 m_out.appendTo(continuation, lastNext);
8034 }
8035
8036 void speculateNotCell(Edge edge)
8037 {
8038 if (!m_interpreter.needsTypeCheck(edge))
8039 return;
8040
8041 LValue value = lowJSValue(edge, ManualOperandSpeculation);
8042 typeCheck(jsValueValue(value), edge, ~SpecCell, isCell(value));
8043 }
8044
8045 void speculateOther(Edge edge)
8046 {
8047 if (!m_interpreter.needsTypeCheck(edge))
8048 return;
8049
8050 LValue value = lowJSValue(edge, ManualOperandSpeculation);
8051 typeCheck(jsValueValue(value), edge, SpecOther, isNotOther(value));
8052 }
8053
8054 void speculateMisc(Edge edge)
8055 {
8056 if (!m_interpreter.needsTypeCheck(edge))
8057 return;
8058
8059 LValue value = lowJSValue(edge, ManualOperandSpeculation);
8060 typeCheck(jsValueValue(value), edge, SpecMisc, isNotMisc(value));
8061 }
8062
8063 bool masqueradesAsUndefinedWatchpointIsStillValid()
8064 {
8065 return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->origin.semantic);
8066 }
8067
8068 LValue loadMarkByte(LValue base)
8069 {
8070 return m_out.load8(base, m_heaps.JSCell_gcData);
8071 }
8072
8073 void emitStoreBarrier(LValue base)
8074 {
8075#if ENABLE(GGC)
8076 LBasicBlock isMarkedAndNotRemembered = FTL_NEW_BLOCK(m_out, ("Store barrier is marked block"));
8077 LBasicBlock bufferHasSpace = FTL_NEW_BLOCK(m_out, ("Store barrier buffer has space"));
8078 LBasicBlock bufferIsFull = FTL_NEW_BLOCK(m_out, ("Store barrier buffer is full"));
8079 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Store barrier continuation"));
8080
8081 // Check the mark byte.
8082 m_out.branch(
8083 m_out.notZero8(loadMarkByte(base)), usually(continuation), rarely(isMarkedAndNotRemembered));
8084
8085 // Append to the write barrier buffer.
8086 LBasicBlock lastNext = m_out.appendTo(isMarkedAndNotRemembered, bufferHasSpace);
8087 LValue currentBufferIndex = m_out.load32(m_out.absolute(vm().heap.writeBarrierBuffer().currentIndexAddress()));
8088 LValue bufferCapacity = m_out.constInt32(vm().heap.writeBarrierBuffer().capacity());
8089 m_out.branch(
8090 m_out.lessThan(currentBufferIndex, bufferCapacity),
8091 usually(bufferHasSpace), rarely(bufferIsFull));
8092
8093 // Buffer has space, store to it.
8094 m_out.appendTo(bufferHasSpace, bufferIsFull);
8095 LValue writeBarrierBufferBase = m_out.constIntPtr(vm().heap.writeBarrierBuffer().buffer());
8096 m_out.storePtr(base, m_out.baseIndex(m_heaps.WriteBarrierBuffer_bufferContents, writeBarrierBufferBase, m_out.zeroExtPtr(currentBufferIndex)));
8097 m_out.store32(m_out.add(currentBufferIndex, m_out.constInt32(1)), m_out.absolute(vm().heap.writeBarrierBuffer().currentIndexAddress()));
8098 m_out.jump(continuation);
8099
8100 // Buffer is out of space, flush it.
8101 m_out.appendTo(bufferIsFull, continuation);
8102 vmCallNoExceptions(m_out.operation(operationFlushWriteBarrierBuffer), m_callFrame, base);
8103 m_out.jump(continuation);
8104
8105 m_out.appendTo(continuation, lastNext);
8106#else
8107 UNUSED_PARAM(base);
8108#endif
8109 }
8110
8111 template<typename... Args>
8112 LValue vmCall(LValue function, Args... args)
8113 {
8114 callPreflight();
8115 LValue result = m_out.call(function, args...);
8116 callCheck();
8117 return result;
8118 }
8119
8120 template<typename... Args>
8121 LValue vmCallNoExceptions(LValue function, Args... args)
8122 {
8123 callPreflight();
8124 LValue result = m_out.call(function, args...);
8125 return result;
8126 }
8127
8128 void callPreflight(CodeOrigin codeOrigin)
8129 {
8130 m_out.store32(
8131 m_out.constInt32(
8132 CallFrame::Location::encodeAsCodeOriginIndex(
8133 m_ftlState.jitCode->common.addCodeOrigin(codeOrigin))),
8134 tagFor(JSStack::ArgumentCount));
8135 }
8136 void callPreflight()
8137 {
8138 callPreflight(m_node->origin.semantic);
8139 }
8140
8141 void callCheck()
8142 {
8143 if (Options::enableExceptionFuzz())
8144 m_out.call(m_out.operation(operationExceptionFuzz));
8145
8146 LBasicBlock continuation = FTL_NEW_BLOCK(m_out, ("Exception check continuation"));
8147
8148 LValue exception = m_out.load64(m_out.absolute(vm().addressOfException()));
8149
8150 m_out.branch(
8151 m_out.notZero64(exception), rarely(m_handleExceptions), usually(continuation));
8152
8153 m_out.appendTo(continuation);
8154 }
8155
8156 LBasicBlock lowBlock(BasicBlock* block)
8157 {
8158 return m_blocks.get(block);
8159 }
8160
8161 void appendOSRExit(
8162 ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition)
8163 {
8164 if (verboseCompilationEnabled()) {
8165 dataLog(" OSR exit #", m_ftlState.jitCode->osrExit.size(), " with availability: ", availabilityMap(), "\n");
8166 if (!m_availableRecoveries.isEmpty())
8167 dataLog(" Available recoveries: ", listDump(m_availableRecoveries), "\n");
8168 }
8169
8170 if (doOSRExitFuzzing()) {
8171 LValue numberOfFuzzChecks = m_out.add(
8172 m_out.load32(m_out.absolute(&g_numberOfOSRExitFuzzChecks)),
8173 m_out.int32One);
8174
8175 m_out.store32(numberOfFuzzChecks, m_out.absolute(&g_numberOfOSRExitFuzzChecks));
8176
8177 if (unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter()) {
8178 failCondition = m_out.bitOr(
8179 failCondition,
8180 m_out.aboveOrEqual(numberOfFuzzChecks, m_out.constInt32(atOrAfter)));
8181 }
8182 if (unsigned at = Options::fireOSRExitFuzzAt()) {
8183 failCondition = m_out.bitOr(
8184 failCondition,
8185 m_out.equal(numberOfFuzzChecks, m_out.constInt32(at)));
8186 }
8187 }
8188
8189 ASSERT(m_ftlState.jitCode->osrExit.size() == m_ftlState.finalizer->osrExit.size());
8190
8191 m_ftlState.jitCode->osrExit.append(OSRExit(
8192 kind, lowValue.format(), m_graph.methodOfGettingAValueProfileFor(highValue),
8193 m_codeOriginForExitTarget, m_codeOriginForExitProfile,
8194 availabilityMap().m_locals.numberOfArguments(),
8195 availabilityMap().m_locals.numberOfLocals()));
8196 m_ftlState.finalizer->osrExit.append(OSRExitCompilationInfo());
8197
8198 OSRExit& exit = m_ftlState.jitCode->osrExit.last();
8199
8200 LBasicBlock lastNext = nullptr;
8201 LBasicBlock continuation = nullptr;
8202
8203 LBasicBlock failCase = FTL_NEW_BLOCK(m_out, ("OSR exit failCase for ", m_node));
8204 continuation = FTL_NEW_BLOCK(m_out, ("OSR exit continuation for ", m_node));
8205
8206 m_out.branch(failCondition, rarely(failCase), usually(continuation));
8207
8208 lastNext = m_out.appendTo(failCase, continuation);
8209
8210 emitOSRExitCall(exit, lowValue);
8211
8212 m_out.unreachable();
8213
8214 m_out.appendTo(continuation, lastNext);
8215 }
8216
8217 void emitOSRExitCall(OSRExit& exit, FormattedValue lowValue)
8218 {
8219 ExitArgumentList arguments;
8220
8221 CodeOrigin codeOrigin = exit.m_codeOrigin;
8222
8223 buildExitArguments(exit, arguments, lowValue, codeOrigin);
8224
8225 callStackmap(exit, arguments);
8226 }
8227
8228 void buildExitArguments(
8229 OSRExit& exit, ExitArgumentList& arguments, FormattedValue lowValue,
8230 CodeOrigin codeOrigin)
8231 {
8232 if (!!lowValue)
8233 arguments.append(lowValue.value());
8234
8235 AvailabilityMap availabilityMap = this->availabilityMap();
8236 availabilityMap.pruneByLiveness(m_graph, codeOrigin);
8237
8238 HashMap<Node*, ExitTimeObjectMaterialization*> map;
8239 availabilityMap.forEachAvailability(
8240 [&] (Availability availability) {
8241 if (!availability.shouldUseNode())
8242 return;
8243
8244 Node* node = availability.node();
8245 if (!node->isPhantomAllocation())
8246 return;
8247
8248 auto result = map.add(node, nullptr);
8249 if (result.isNewEntry) {
8250 result.iterator->value =
8251 exit.m_materializations.add(node->op(), node->origin.semantic);
8252 }
8253 });
8254
8255 for (unsigned i = 0; i < exit.m_values.size(); ++i) {
8256 int operand = exit.m_values.operandForIndex(i);
8257
8258 Availability availability = availabilityMap.m_locals[i];
8259
8260 if (Options::validateFTLOSRExitLiveness()) {
8261 DFG_ASSERT(
8262 m_graph, m_node,
8263 (!(availability.isDead() && m_graph.isLiveInBytecode(VirtualRegister(operand), codeOrigin))) || m_graph.m_plan.mode == FTLForOSREntryMode);
8264 }
8265
8266 exit.m_values[i] = exitValueForAvailability(arguments, map, availability);
8267 }
8268
8269 for (auto heapPair : availabilityMap.m_heap) {
8270 Node* node = heapPair.key.base();
8271 ExitTimeObjectMaterialization* materialization = map.get(node);
8272 materialization->add(
8273 heapPair.key.descriptor(),
8274 exitValueForAvailability(arguments, map, heapPair.value));
8275 }
8276
8277 if (verboseCompilationEnabled()) {
8278 dataLog(" Exit values: ", exit.m_values, "\n");
8279 if (!exit.m_materializations.isEmpty()) {
8280 dataLog(" Materializations: \n");
8281 for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
8282 dataLog(" ", pointerDump(materialization), "\n");
8283 }
8284 }
8285 }
8286
8287 void callStackmap(OSRExit& exit, ExitArgumentList& arguments)
8288 {
8289 exit.m_stackmapID = m_stackmapIDs++;
8290 arguments.insert(0, m_out.constInt32(MacroAssembler::maxJumpReplacementSize()));
8291 arguments.insert(0, m_out.constInt64(exit.m_stackmapID));
8292
8293 m_out.call(m_out.stackmapIntrinsic(), arguments);
8294 }
8295
8296 ExitValue exitValueForAvailability(
8297 ExitArgumentList& arguments, const HashMap<Node*, ExitTimeObjectMaterialization*>& map,
8298 Availability availability)
8299 {
8300 FlushedAt flush = availability.flushedAt();
8301 switch (flush.format()) {
8302 case DeadFlush:
8303 case ConflictingFlush:
8304 if (availability.hasNode())
8305 return exitValueForNode(arguments, map, availability.node());
8306
8307 // This means that the value is dead. It could be dead in bytecode or it could have
8308 // been killed by our DCE, which can sometimes kill things even if they were live in
8309 // bytecode.
8310 return ExitValue::dead();
8311
8312 case FlushedJSValue:
8313 case FlushedCell:
8314 case FlushedBoolean:
8315 return ExitValue::inJSStack(flush.virtualRegister());
8316
8317 case FlushedInt32:
8318 return ExitValue::inJSStackAsInt32(flush.virtualRegister());
8319
8320 case FlushedInt52:
8321 return ExitValue::inJSStackAsInt52(flush.virtualRegister());
8322
8323 case FlushedDouble:
8324 return ExitValue::inJSStackAsDouble(flush.virtualRegister());
8325 }
8326
8327 DFG_CRASH(m_graph, m_node, "Invalid flush format");
8328 return ExitValue::dead();
8329 }
8330
8331 ExitValue exitValueForNode(
8332 ExitArgumentList& arguments, const HashMap<Node*, ExitTimeObjectMaterialization*>& map,
8333 Node* node)
8334 {
8335 ASSERT(node->shouldGenerate());
8336 ASSERT(node->hasResult());
8337
8338 if (node) {
8339 switch (node->op()) {
8340 case BottomValue:
8341 // This might arise in object materializations. I actually doubt that it would,
8342 // but it seems worthwhile to be conservative.
8343 return ExitValue::dead();
8344
8345 case JSConstant:
8346 case Int52Constant:
8347 case DoubleConstant:
8348 return ExitValue::constant(node->asJSValue());
8349
8350 default:
8351 if (node->isPhantomAllocation())
8352 return ExitValue::materializeNewObject(map.get(node));
8353 break;
8354 }
8355 }
8356
8357 for (unsigned i = 0; i < m_availableRecoveries.size(); ++i) {
8358 AvailableRecovery recovery = m_availableRecoveries[i];
8359 if (recovery.node() != node)
8360 continue;
8361
8362 ExitValue result = ExitValue::recovery(
8363 recovery.opcode(), arguments.size(), arguments.size() + 1,
8364 recovery.format());
8365 arguments.append(recovery.left());
8366 arguments.append(recovery.right());
8367 return result;
8368 }
8369
8370 LoweredNodeValue value = m_int32Values.get(node);
8371 if (isValid(value))
8372 return exitArgument(arguments, ValueFormatInt32, value.value());
8373
8374 value = m_int52Values.get(node);
8375 if (isValid(value))
8376 return exitArgument(arguments, ValueFormatInt52, value.value());
8377
8378 value = m_strictInt52Values.get(node);
8379 if (isValid(value))
8380 return exitArgument(arguments, ValueFormatStrictInt52, value.value());
8381
8382 value = m_booleanValues.get(node);
8383 if (isValid(value)) {
8384 LValue valueToPass = m_out.zeroExt(value.value(), m_out.int32);
8385 return exitArgument(arguments, ValueFormatBoolean, valueToPass);
8386 }
8387
8388 value = m_jsValueValues.get(node);
8389 if (isValid(value))
8390 return exitArgument(arguments, ValueFormatJSValue, value.value());
8391
8392 value = m_doubleValues.get(node);
8393 if (isValid(value))
8394 return exitArgument(arguments, ValueFormatDouble, value.value());
8395
8396 DFG_CRASH(m_graph, m_node, toCString("Cannot find value for node: ", node).data());
8397 return ExitValue::dead();
8398 }
8399
8400 ExitValue exitArgument(ExitArgumentList& arguments, ValueFormat format, LValue value)
8401 {
8402 ExitValue result = ExitValue::exitArgument(ExitArgument(format, arguments.size()));
8403 arguments.append(value);
8404 return result;
8405 }
8406
8407 bool doesKill(Edge edge)
8408 {
8409 if (edge.doesNotKill())
8410 return false;
8411
8412 if (edge->hasConstant())
8413 return false;
8414
8415 return true;
8416 }
8417
8418 void addAvailableRecovery(
8419 Node* node, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format)
8420 {
8421 m_availableRecoveries.append(AvailableRecovery(node, opcode, left, right, format));
8422 }
8423
8424 void addAvailableRecovery(
8425 Edge edge, RecoveryOpcode opcode, LValue left, LValue right, ValueFormat format)
8426 {
8427 addAvailableRecovery(edge.node(), opcode, left, right, format);
8428 }
8429
8430 void setInt32(Node* node, LValue value)
8431 {
8432 m_int32Values.set(node, LoweredNodeValue(value, m_highBlock));
8433 }
8434 void setInt52(Node* node, LValue value)
8435 {
8436 m_int52Values.set(node, LoweredNodeValue(value, m_highBlock));
8437 }
8438 void setStrictInt52(Node* node, LValue value)
8439 {
8440 m_strictInt52Values.set(node, LoweredNodeValue(value, m_highBlock));
8441 }
8442 void setInt52(Node* node, LValue value, Int52Kind kind)
8443 {
8444 switch (kind) {
8445 case Int52:
8446 setInt52(node, value);
8447 return;
8448
8449 case StrictInt52:
8450 setStrictInt52(node, value);
8451 return;
8452 }
8453
8454 DFG_CRASH(m_graph, m_node, "Corrupt int52 kind");
8455 }
8456 void setJSValue(Node* node, LValue value)
8457 {
8458 m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock));
8459 }
8460 void setBoolean(Node* node, LValue value)
8461 {
8462 m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock));
8463 }
8464 void setStorage(Node* node, LValue value)
8465 {
8466 m_storageValues.set(node, LoweredNodeValue(value, m_highBlock));
8467 }
8468 void setDouble(Node* node, LValue value)
8469 {
8470 m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock));
8471 }
8472
8473 void setInt32(LValue value)
8474 {
8475 setInt32(m_node, value);
8476 }
8477 void setInt52(LValue value)
8478 {
8479 setInt52(m_node, value);
8480 }
8481 void setStrictInt52(LValue value)
8482 {
8483 setStrictInt52(m_node, value);
8484 }
8485 void setInt52(LValue value, Int52Kind kind)
8486 {
8487 setInt52(m_node, value, kind);
8488 }
8489 void setJSValue(LValue value)
8490 {
8491 setJSValue(m_node, value);
8492 }
8493 void setBoolean(LValue value)
8494 {
8495 setBoolean(m_node, value);
8496 }
8497 void setStorage(LValue value)
8498 {
8499 setStorage(m_node, value);
8500 }
8501 void setDouble(LValue value)
8502 {
8503 setDouble(m_node, value);
8504 }
8505
8506 bool isValid(const LoweredNodeValue& value)
8507 {
8508 if (!value)
8509 return false;
8510 if (!m_graph.m_dominators.dominates(value.block(), m_highBlock))
8511 return false;
8512 return true;
8513 }
8514
8515 void addWeakReference(JSCell* target)
8516 {
8517 m_graph.m_plan.weakReferences.addLazily(target);
8518 }
8519
8520 LValue loadStructure(LValue value)
8521 {
8522 LValue tableIndex = m_out.load32(value, m_heaps.JSCell_structureID);
8523 LValue tableBase = m_out.loadPtr(
8524 m_out.absolute(vm().heap.structureIDTable().base()));
8525 TypedPointer address = m_out.baseIndex(
8526 m_heaps.structureTable, tableBase, m_out.zeroExtPtr(tableIndex));
8527 return m_out.loadPtr(address);
8528 }
8529
8530 LValue weakPointer(JSCell* pointer)
8531 {
8532 addWeakReference(pointer);
8533 return m_out.constIntPtr(pointer);
8534 }
8535
8536 LValue weakStructureID(Structure* structure)
8537 {
8538 addWeakReference(structure);
8539 return m_out.constInt32(structure->id());
8540 }
8541
8542 LValue weakStructure(Structure* structure)
8543 {
8544 return weakPointer(structure);
8545 }
8546
8547 TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0)
8548 {
8549 return m_out.address(base, m_heaps.variables[operand], offset);
8550 }
8551 TypedPointer payloadFor(LValue base, int operand)
8552 {
8553 return addressFor(base, operand, PayloadOffset);
8554 }
8555 TypedPointer tagFor(LValue base, int operand)
8556 {
8557 return addressFor(base, operand, TagOffset);
8558 }
8559 TypedPointer addressFor(int operand, ptrdiff_t offset = 0)
8560 {
8561 return addressFor(VirtualRegister(operand), offset);
8562 }
8563 TypedPointer addressFor(VirtualRegister operand, ptrdiff_t offset = 0)
8564 {
8565 if (operand.isLocal())
8566 return addressFor(m_captured, operand.offset(), offset);
8567 return addressFor(m_callFrame, operand.offset(), offset);
8568 }
8569 TypedPointer payloadFor(int operand)
8570 {
8571 return payloadFor(VirtualRegister(operand));
8572 }
8573 TypedPointer payloadFor(VirtualRegister operand)
8574 {
8575 return addressFor(operand, PayloadOffset);
8576 }
8577 TypedPointer tagFor(int operand)
8578 {
8579 return tagFor(VirtualRegister(operand));
8580 }
8581 TypedPointer tagFor(VirtualRegister operand)
8582 {
8583 return addressFor(operand, TagOffset);
8584 }
8585
8586 AbstractValue abstractValue(Node* node)
8587 {
8588 return m_state.forNode(node);
8589 }
8590 AbstractValue abstractValue(Edge edge)
8591 {
8592 return abstractValue(edge.node());
8593 }
8594
8595 SpeculatedType provenType(Node* node)
8596 {
8597 return abstractValue(node).m_type;
8598 }
8599 SpeculatedType provenType(Edge edge)
8600 {
8601 return provenType(edge.node());
8602 }
8603
8604 JSValue provenValue(Node* node)
8605 {
8606 return abstractValue(node).m_value;
8607 }
8608 JSValue provenValue(Edge edge)
8609 {
8610 return provenValue(edge.node());
8611 }
8612
8613 StructureAbstractValue abstractStructure(Node* node)
8614 {
8615 return abstractValue(node).m_structure;
8616 }
8617 StructureAbstractValue abstractStructure(Edge edge)
8618 {
8619 return abstractStructure(edge.node());
8620 }
8621
8622 void crash()
8623 {
8624 crash(m_highBlock->index, m_node->index());
8625 }
8626 void crash(BlockIndex blockIndex, unsigned nodeIndex)
8627 {
8628#if ASSERT_DISABLED
8629 m_out.call(m_out.operation(ftlUnreachable));
8630 UNUSED_PARAM(blockIndex);
8631 UNUSED_PARAM(nodeIndex);
8632#else
8633 m_out.call(
8634 m_out.intToPtr(
8635 m_out.constIntPtr(ftlUnreachable),
8636 pointerType(
8637 functionType(
8638 m_out.voidType, m_out.intPtr, m_out.int32, m_out.int32))),
8639 m_out.constIntPtr(codeBlock()), m_out.constInt32(blockIndex),
8640 m_out.constInt32(nodeIndex));
8641#endif
8642 m_out.unreachable();
8643 }
8644
8645 AvailabilityMap& availabilityMap() { return m_availabilityCalculator.m_availability; }
8646
8647 VM& vm() { return m_graph.m_vm; }
8648 CodeBlock* codeBlock() { return m_graph.m_codeBlock; }
8649
8650 Graph& m_graph;
8651 State& m_ftlState;
8652 AbstractHeapRepository m_heaps;
8653 Output m_out;
8654
8655 LBasicBlock m_prologue;
8656 LBasicBlock m_handleExceptions;
8657 HashMap<BasicBlock*, LBasicBlock> m_blocks;
8658
8659 LValue m_execState;
8660 LValue m_execStorage;
8661 LValue m_callFrame;
8662 LValue m_captured;
8663 LValue m_tagTypeNumber;
8664 LValue m_tagMask;
8665
8666 HashMap<Node*, LoweredNodeValue> m_int32Values;
8667 HashMap<Node*, LoweredNodeValue> m_strictInt52Values;
8668 HashMap<Node*, LoweredNodeValue> m_int52Values;
8669 HashMap<Node*, LoweredNodeValue> m_jsValueValues;
8670 HashMap<Node*, LoweredNodeValue> m_booleanValues;
8671 HashMap<Node*, LoweredNodeValue> m_storageValues;
8672 HashMap<Node*, LoweredNodeValue> m_doubleValues;
8673
8674 // This is a bit of a hack. It prevents LLVM from having to do CSE on loading of arguments.
8675 // It's nice to have these optimizations on our end because we can guarantee them a bit better.
8676 // Probably also saves LLVM compile time.
8677 HashMap<Node*, LValue> m_loadedArgumentValues;
8678
8679 HashMap<Node*, LValue> m_phis;
8680
8681 LocalOSRAvailabilityCalculator m_availabilityCalculator;
8682
8683 Vector<AvailableRecovery, 3> m_availableRecoveries;
8684
8685 InPlaceAbstractState m_state;
8686 AbstractInterpreter<InPlaceAbstractState> m_interpreter;
8687 BasicBlock* m_highBlock;
8688 BasicBlock* m_nextHighBlock;
8689 LBasicBlock m_nextLowBlock;
8690
8691 CodeOrigin m_codeOriginForExitTarget;
8692 CodeOrigin m_codeOriginForExitProfile;
8693 unsigned m_nodeIndex;
8694 Node* m_node;
8695
8696 uint32_t m_stackmapIDs;
8697 unsigned m_tbaaKind;
8698 unsigned m_tbaaStructKind;
8699};
8700
8701} // anonymous namespace
8702
8703void lowerDFGToLLVM(State& state)
8704{
8705 LowerDFGToLLVM lowering(state);
8706 lowering.lower();
8707}
8708
8709} } // namespace JSC::FTL
8710
8711#endif // ENABLE(FTL_JIT)
8712
mirror of JavaScriptCore