Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 : * vim: set ts=8 sts=4 et sw=4 tw=99:
3 : * This Source Code Form is subject to the terms of the Mozilla Public
4 : * License, v. 2.0. If a copy of the MPL was not distributed with this
5 : * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 :
7 : #include "jit/CodeGenerator.h"
8 :
9 : #include "mozilla/Assertions.h"
10 : #include "mozilla/Attributes.h"
11 : #include "mozilla/Casting.h"
12 : #include "mozilla/DebugOnly.h"
13 : #include "mozilla/EnumeratedArray.h"
14 : #include "mozilla/EnumeratedRange.h"
15 : #include "mozilla/MathAlgorithms.h"
16 : #include "mozilla/ScopeExit.h"
17 : #include "mozilla/Unused.h"
18 :
19 : #include <type_traits>
20 :
21 : #include "jslibmath.h"
22 : #include "jsmath.h"
23 : #include "jsnum.h"
24 :
25 : #include "builtin/Eval.h"
26 : #include "builtin/RegExp.h"
27 : #include "builtin/SelfHostingDefines.h"
28 : #include "builtin/String.h"
29 : #include "builtin/TypedObject.h"
30 : #include "gc/Nursery.h"
31 : #include "irregexp/NativeRegExpMacroAssembler.h"
32 : #include "jit/AtomicOperations.h"
33 : #include "jit/BaselineCompiler.h"
34 : #include "jit/IonBuilder.h"
35 : #include "jit/IonIC.h"
36 : #include "jit/IonOptimizationLevels.h"
37 : #include "jit/JitcodeMap.h"
38 : #include "jit/JitSpewer.h"
39 : #include "jit/Linker.h"
40 : #include "jit/Lowering.h"
41 : #include "jit/MIRGenerator.h"
42 : #include "jit/MoveEmitter.h"
43 : #include "jit/RangeAnalysis.h"
44 : #include "jit/SharedICHelpers.h"
45 : #include "jit/StackSlotAllocator.h"
46 : #include "jit/VMFunctions.h"
47 : #include "util/Unicode.h"
48 : #include "vm/AsyncFunction.h"
49 : #include "vm/AsyncIteration.h"
50 : #include "vm/MatchPairs.h"
51 : #include "vm/RegExpObject.h"
52 : #include "vm/RegExpStatics.h"
53 : #include "vm/StringType.h"
54 : #include "vm/TraceLogging.h"
55 : #include "vm/TypedArrayObject.h"
56 : #include "vtune/VTuneWrapper.h"
57 :
58 : #include "builtin/Boolean-inl.h"
59 : #include "jit/MacroAssembler-inl.h"
60 : #include "jit/shared/CodeGenerator-shared-inl.h"
61 : #include "jit/shared/Lowering-shared-inl.h"
62 : #include "jit/TemplateObject-inl.h"
63 : #include "vm/Interpreter-inl.h"
64 :
65 : using namespace js;
66 : using namespace js::jit;
67 :
68 : using mozilla::AssertedCast;
69 : using mozilla::DebugOnly;
70 : using mozilla::FloatingPoint;
71 : using mozilla::Maybe;
72 : using mozilla::NegativeInfinity;
73 : using mozilla::PositiveInfinity;
74 : using JS::GenericNaN;
75 :
76 : namespace js {
77 : namespace jit {
78 :
79 : class OutOfLineICFallback : public OutOfLineCodeBase<CodeGenerator>
80 : {
81 : private:
82 : LInstruction* lir_;
83 : size_t cacheIndex_;
84 : size_t cacheInfoIndex_;
85 :
86 : public:
87 : OutOfLineICFallback(LInstruction* lir, size_t cacheIndex, size_t cacheInfoIndex)
88 389 : : lir_(lir),
89 : cacheIndex_(cacheIndex),
90 778 : cacheInfoIndex_(cacheInfoIndex)
91 : { }
92 :
93 389 : void bind(MacroAssembler* masm) override {
94 : // The binding of the initial jump is done in
95 : // CodeGenerator::visitOutOfLineICFallback.
96 389 : }
97 :
98 : size_t cacheIndex() const {
99 : return cacheIndex_;
100 : }
101 : size_t cacheInfoIndex() const {
102 : return cacheInfoIndex_;
103 : }
104 : LInstruction* lir() const {
105 : return lir_;
106 : }
107 :
108 0 : void accept(CodeGenerator* codegen) override {
109 0 : codegen->visitOutOfLineICFallback(this);
110 389 : }
111 : };
112 :
113 : void
114 389 : CodeGeneratorShared::addIC(LInstruction* lir, size_t cacheIndex)
115 : {
116 0 : if (cacheIndex == SIZE_MAX) {
117 0 : masm.setOOM();
118 0 : return;
119 : }
120 :
121 0 : DataPtr<IonIC> cache(this, cacheIndex);
122 0 : MInstruction* mir = lir->mirRaw()->toInstruction();
123 0 : if (mir->resumePoint()) {
124 0 : cache->setScriptedLocation(mir->block()->info().script(),
125 383 : mir->resumePoint()->pc());
126 : } else {
127 6 : cache->setIdempotent();
128 : }
129 :
130 0 : Register temp = cache->scratchRegisterForEntryJump();
131 0 : icInfo_.back().icOffsetForJump = masm.movWithPatch(ImmWord(-1), temp);
132 389 : masm.jump(Address(temp, 0));
133 :
134 778 : MOZ_ASSERT(!icInfo_.empty());
135 :
136 0 : OutOfLineICFallback* ool = new(alloc()) OutOfLineICFallback(lir, cacheIndex, icInfo_.length() - 1);
137 389 : addOutOfLineCode(ool, mir);
138 :
139 0 : masm.bind(ool->rejoin());
140 1167 : cache->setRejoinLabel(CodeOffset(ool->rejoin()->offset()));
141 : }
142 :
143 : typedef bool (*IonGetPropertyICFn)(JSContext*, HandleScript, IonGetPropertyIC*, HandleValue, HandleValue,
144 : MutableHandleValue);
145 0 : static const VMFunction IonGetPropertyICInfo =
146 3 : FunctionInfo<IonGetPropertyICFn>(IonGetPropertyIC::update, "IonGetPropertyIC::update");
147 :
148 : typedef bool (*IonSetPropertyICFn)(JSContext*, HandleScript, IonSetPropertyIC*, HandleObject,
149 : HandleValue, HandleValue);
150 0 : static const VMFunction IonSetPropertyICInfo =
151 3 : FunctionInfo<IonSetPropertyICFn>(IonSetPropertyIC::update, "IonSetPropertyIC::update");
152 :
153 : typedef bool (*IonGetPropSuperICFn)(JSContext*, HandleScript, IonGetPropSuperIC*, HandleObject, HandleValue,
154 : HandleValue, MutableHandleValue);
155 0 : static const VMFunction IonGetPropSuperICInfo =
156 3 : FunctionInfo<IonGetPropSuperICFn>(IonGetPropSuperIC::update, "IonGetPropSuperIC::update");
157 :
158 : typedef bool (*IonGetNameICFn)(JSContext*, HandleScript, IonGetNameIC*, HandleObject,
159 : MutableHandleValue);
160 0 : static const VMFunction IonGetNameICInfo =
161 3 : FunctionInfo<IonGetNameICFn>(IonGetNameIC::update, "IonGetNameIC::update");
162 :
163 : typedef bool (*IonHasOwnICFn)(JSContext*, HandleScript, IonHasOwnIC*, HandleValue, HandleValue,
164 : int32_t*);
165 0 : static const VMFunction IonHasOwnICInfo =
166 3 : FunctionInfo<IonHasOwnICFn>(IonHasOwnIC::update, "IonHasOwnIC::update");
167 :
168 : typedef JSObject* (*IonBindNameICFn)(JSContext*, HandleScript, IonBindNameIC*, HandleObject);
169 0 : static const VMFunction IonBindNameICInfo =
170 3 : FunctionInfo<IonBindNameICFn>(IonBindNameIC::update, "IonBindNameIC::update");
171 :
172 : typedef JSObject* (*IonGetIteratorICFn)(JSContext*, HandleScript, IonGetIteratorIC*, HandleValue);
173 0 : static const VMFunction IonGetIteratorICInfo =
174 3 : FunctionInfo<IonGetIteratorICFn>(IonGetIteratorIC::update, "IonGetIteratorIC::update");
175 :
176 : typedef bool (*IonInICFn)(JSContext*, HandleScript, IonInIC*, HandleValue, HandleObject, bool*);
177 0 : static const VMFunction IonInICInfo =
178 3 : FunctionInfo<IonInICFn>(IonInIC::update, "IonInIC::update");
179 :
180 : typedef bool (*IonInstanceOfICFn)(JSContext*, HandleScript, IonInstanceOfIC*,
181 : HandleValue lhs, HandleObject rhs, bool* res);
182 0 : static const VMFunction IonInstanceOfInfo =
183 3 : FunctionInfo<IonInstanceOfICFn>(IonInstanceOfIC::update, "IonInstanceOfIC::update");
184 :
185 : typedef bool (*IonUnaryArithICFn)(JSContext* cx, HandleScript outerScript, IonUnaryArithIC* stub,
186 : HandleValue val, MutableHandleValue res);
187 0 : static const VMFunction IonUnaryArithICInfo =
188 3 : FunctionInfo<IonUnaryArithICFn>(IonUnaryArithIC::update, "IonUnaryArithIC::update");
189 :
190 : void
191 389 : CodeGenerator::visitOutOfLineICFallback(OutOfLineICFallback* ool)
192 : {
193 0 : LInstruction* lir = ool->lir();
194 0 : size_t cacheIndex = ool->cacheIndex();
195 389 : size_t cacheInfoIndex = ool->cacheInfoIndex();
196 :
197 778 : DataPtr<IonIC> ic(this, cacheIndex);
198 :
199 : // Register the location of the OOL path in the IC.
200 1556 : ic->setFallbackLabel(masm.labelForPatch());
201 :
202 389 : switch (ic->kind()) {
203 : case CacheKind::GetProp:
204 : case CacheKind::GetElem: {
205 271 : IonGetPropertyIC* getPropIC = ic->asGetPropertyIC();
206 :
207 271 : saveLive(lir);
208 :
209 0 : pushArg(getPropIC->id());
210 0 : pushArg(getPropIC->value());
211 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
212 542 : pushArg(ImmGCPtr(gen->info().script()));
213 :
214 271 : callVM(IonGetPropertyICInfo, lir);
215 :
216 0 : StoreValueTo(getPropIC->output()).generate(this);
217 1084 : restoreLiveIgnore(lir, StoreValueTo(getPropIC->output()).clobbered());
218 :
219 542 : masm.jump(ool->rejoin());
220 : return;
221 : }
222 : case CacheKind::GetPropSuper:
223 : case CacheKind::GetElemSuper: {
224 2 : IonGetPropSuperIC* getPropSuperIC = ic->asGetPropSuperIC();
225 :
226 2 : saveLive(lir);
227 :
228 0 : pushArg(getPropSuperIC->id());
229 0 : pushArg(getPropSuperIC->receiver());
230 0 : pushArg(getPropSuperIC->object());
231 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
232 4 : pushArg(ImmGCPtr(gen->info().script()));
233 :
234 2 : callVM(IonGetPropSuperICInfo, lir);
235 :
236 0 : StoreValueTo(getPropSuperIC->output()).generate(this);
237 8 : restoreLiveIgnore(lir, StoreValueTo(getPropSuperIC->output()).clobbered());
238 :
239 4 : masm.jump(ool->rejoin());
240 : return;
241 : }
242 : case CacheKind::SetProp:
243 : case CacheKind::SetElem: {
244 42 : IonSetPropertyIC* setPropIC = ic->asSetPropertyIC();
245 :
246 42 : saveLive(lir);
247 :
248 0 : pushArg(setPropIC->rhs());
249 0 : pushArg(setPropIC->id());
250 0 : pushArg(setPropIC->object());
251 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
252 84 : pushArg(ImmGCPtr(gen->info().script()));
253 :
254 42 : callVM(IonSetPropertyICInfo, lir);
255 :
256 42 : restoreLive(lir);
257 :
258 84 : masm.jump(ool->rejoin());
259 : return;
260 : }
261 : case CacheKind::GetName: {
262 53 : IonGetNameIC* getNameIC = ic->asGetNameIC();
263 :
264 53 : saveLive(lir);
265 :
266 0 : pushArg(getNameIC->environment());
267 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
268 106 : pushArg(ImmGCPtr(gen->info().script()));
269 :
270 53 : callVM(IonGetNameICInfo, lir);
271 :
272 0 : StoreValueTo(getNameIC->output()).generate(this);
273 159 : restoreLiveIgnore(lir, StoreValueTo(getNameIC->output()).clobbered());
274 :
275 106 : masm.jump(ool->rejoin());
276 : return;
277 : }
278 : case CacheKind::BindName: {
279 0 : IonBindNameIC* bindNameIC = ic->asBindNameIC();
280 :
281 0 : saveLive(lir);
282 :
283 0 : pushArg(bindNameIC->environment());
284 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
285 0 : pushArg(ImmGCPtr(gen->info().script()));
286 :
287 0 : callVM(IonBindNameICInfo, lir);
288 :
289 0 : StoreRegisterTo(bindNameIC->output()).generate(this);
290 0 : restoreLiveIgnore(lir, StoreRegisterTo(bindNameIC->output()).clobbered());
291 :
292 0 : masm.jump(ool->rejoin());
293 : return;
294 : }
295 : case CacheKind::GetIterator: {
296 0 : IonGetIteratorIC* getIteratorIC = ic->asGetIteratorIC();
297 :
298 0 : saveLive(lir);
299 :
300 0 : pushArg(getIteratorIC->value());
301 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
302 0 : pushArg(ImmGCPtr(gen->info().script()));
303 :
304 0 : callVM(IonGetIteratorICInfo, lir);
305 :
306 0 : StoreRegisterTo(getIteratorIC->output()).generate(this);
307 0 : restoreLiveIgnore(lir, StoreRegisterTo(getIteratorIC->output()).clobbered());
308 :
309 0 : masm.jump(ool->rejoin());
310 : return;
311 : }
312 : case CacheKind::In: {
313 20 : IonInIC* inIC = ic->asInIC();
314 :
315 20 : saveLive(lir);
316 :
317 0 : pushArg(inIC->object());
318 0 : pushArg(inIC->key());
319 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
320 40 : pushArg(ImmGCPtr(gen->info().script()));
321 :
322 20 : callVM(IonInICInfo, lir);
323 :
324 0 : StoreRegisterTo(inIC->output()).generate(this);
325 60 : restoreLiveIgnore(lir, StoreRegisterTo(inIC->output()).clobbered());
326 :
327 40 : masm.jump(ool->rejoin());
328 : return;
329 : }
330 : case CacheKind::HasOwn: {
331 1 : IonHasOwnIC* hasOwnIC = ic->asHasOwnIC();
332 :
333 1 : saveLive(lir);
334 :
335 0 : pushArg(hasOwnIC->id());
336 0 : pushArg(hasOwnIC->value());
337 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
338 2 : pushArg(ImmGCPtr(gen->info().script()));
339 :
340 1 : callVM(IonHasOwnICInfo, lir);
341 :
342 0 : StoreRegisterTo(hasOwnIC->output()).generate(this);
343 3 : restoreLiveIgnore(lir, StoreRegisterTo(hasOwnIC->output()).clobbered());
344 :
345 2 : masm.jump(ool->rejoin());
346 : return;
347 : }
348 : case CacheKind::InstanceOf: {
349 0 : IonInstanceOfIC* hasInstanceOfIC = ic->asInstanceOfIC();
350 :
351 0 : saveLive(lir);
352 :
353 0 : pushArg(hasInstanceOfIC->rhs());
354 0 : pushArg(hasInstanceOfIC->lhs());
355 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
356 0 : pushArg(ImmGCPtr(gen->info().script()));
357 :
358 0 : callVM(IonInstanceOfInfo, lir);
359 :
360 0 : StoreRegisterTo(hasInstanceOfIC->output()).generate(this);
361 0 : restoreLiveIgnore(lir, StoreRegisterTo(hasInstanceOfIC->output()).clobbered());
362 :
363 0 : masm.jump(ool->rejoin());
364 : return;
365 : }
366 : case CacheKind::UnaryArith: {
367 0 : IonUnaryArithIC* unaryArithIC = ic->asUnaryArithIC();
368 :
369 0 : saveLive(lir);
370 :
371 0 : pushArg(unaryArithIC->input());
372 0 : icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
373 0 : pushArg(ImmGCPtr(gen->info().script()));
374 0 : callVM(IonUnaryArithICInfo, lir);
375 :
376 0 : StoreValueTo(unaryArithIC->output()).generate(this);
377 0 : restoreLiveIgnore(lir, StoreValueTo(unaryArithIC->output()).clobbered());
378 :
379 0 : masm.jump(ool->rejoin());
380 : return;
381 : }
382 : case CacheKind::Call:
383 : case CacheKind::Compare:
384 : case CacheKind::TypeOf:
385 : case CacheKind::ToBool:
386 : case CacheKind::GetIntrinsic:
387 0 : MOZ_CRASH("Unsupported IC");
388 : }
389 0 : MOZ_CRASH();
390 : }
391 :
392 : StringObject*
393 0 : MNewStringObject::templateObj() const
394 : {
395 0 : return &templateObj_->as<StringObject>();
396 : }
397 :
398 48 : CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
399 : : CodeGeneratorSpecific(gen, graph, masm)
400 : , ionScriptLabels_(gen->alloc())
401 : , scriptCounts_(nullptr)
402 : , simdTemplatesToReadBarrier_(0)
403 144 : , realmStubsToReadBarrier_(0)
404 : {
405 48 : }
406 :
407 141 : CodeGenerator::~CodeGenerator()
408 : {
409 0 : js_delete(scriptCounts_);
410 47 : }
411 :
412 : typedef bool (*StringToNumberFn)(JSContext*, JSString*, double*);
413 0 : static const VMFunction StringToNumberInfo =
414 3 : FunctionInfo<StringToNumberFn>(StringToNumber, "StringToNumber");
415 :
416 : void
417 0 : CodeGenerator::visitValueToInt32(LValueToInt32* lir)
418 : {
419 0 : ValueOperand operand = ToValue(lir, LValueToInt32::Input);
420 0 : Register output = ToRegister(lir->output());
421 0 : FloatRegister temp = ToFloatRegister(lir->tempFloat());
422 :
423 : MDefinition* input;
424 0 : if (lir->mode() == LValueToInt32::NORMAL)
425 0 : input = lir->mirNormal()->input();
426 : else
427 0 : input = lir->mirTruncate()->input();
428 :
429 0 : Label fails;
430 0 : if (lir->mode() == LValueToInt32::TRUNCATE) {
431 0 : OutOfLineCode* oolDouble = oolTruncateDouble(temp, output, lir->mir());
432 :
433 : // We can only handle strings in truncation contexts, like bitwise
434 : // operations.
435 : Label* stringEntry;
436 : Label* stringRejoin;
437 : Register stringReg;
438 0 : if (input->mightBeType(MIRType::String)) {
439 0 : stringReg = ToRegister(lir->temp());
440 0 : OutOfLineCode* oolString = oolCallVM(StringToNumberInfo, lir, ArgList(stringReg),
441 0 : StoreFloatRegisterTo(temp));
442 0 : stringEntry = oolString->entry();
443 0 : stringRejoin = oolString->rejoin();
444 : } else {
445 : stringReg = InvalidReg;
446 : stringEntry = nullptr;
447 : stringRejoin = nullptr;
448 : }
449 :
450 0 : masm.truncateValueToInt32(operand, input, stringEntry, stringRejoin, oolDouble->entry(),
451 0 : stringReg, temp, output, &fails);
452 0 : masm.bind(oolDouble->rejoin());
453 : } else {
454 0 : masm.convertValueToInt32(operand, input, temp, output, &fails,
455 0 : lir->mirNormal()->canBeNegativeZero(),
456 : lir->mirNormal()->conversion());
457 : }
458 :
459 0 : bailoutFrom(&fails, lir->snapshot());
460 0 : }
461 :
462 : void
463 0 : CodeGenerator::visitValueToDouble(LValueToDouble* lir)
464 : {
465 0 : MToDouble* mir = lir->mir();
466 0 : ValueOperand operand = ToValue(lir, LValueToDouble::Input);
467 0 : FloatRegister output = ToFloatRegister(lir->output());
468 :
469 0 : Label isDouble, isInt32, isBool, isNull, isUndefined, done;
470 0 : bool hasBoolean = false, hasNull = false, hasUndefined = false;
471 :
472 : {
473 0 : ScratchTagScope tag(masm, operand);
474 0 : masm.splitTagForTest(operand, tag);
475 :
476 0 : masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
477 0 : masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
478 :
479 0 : if (mir->conversion() != MToFPInstruction::NumbersOnly) {
480 0 : masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
481 0 : masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
482 0 : hasBoolean = true;
483 0 : hasUndefined = true;
484 0 : if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) {
485 0 : masm.branchTestNull(Assembler::Equal, tag, &isNull);
486 0 : hasNull = true;
487 : }
488 : }
489 : }
490 :
491 0 : bailout(lir->snapshot());
492 :
493 0 : if (hasNull) {
494 0 : masm.bind(&isNull);
495 0 : masm.loadConstantDouble(0.0, output);
496 0 : masm.jump(&done);
497 : }
498 :
499 0 : if (hasUndefined) {
500 0 : masm.bind(&isUndefined);
501 0 : masm.loadConstantDouble(GenericNaN(), output);
502 0 : masm.jump(&done);
503 : }
504 :
505 0 : if (hasBoolean) {
506 0 : masm.bind(&isBool);
507 0 : masm.boolValueToDouble(operand, output);
508 0 : masm.jump(&done);
509 : }
510 :
511 0 : masm.bind(&isInt32);
512 0 : masm.int32ValueToDouble(operand, output);
513 0 : masm.jump(&done);
514 :
515 0 : masm.bind(&isDouble);
516 0 : masm.unboxDouble(operand, output);
517 0 : masm.bind(&done);
518 0 : }
519 :
520 : void
521 0 : CodeGenerator::visitValueToFloat32(LValueToFloat32* lir)
522 : {
523 0 : MToFloat32* mir = lir->mir();
524 0 : ValueOperand operand = ToValue(lir, LValueToFloat32::Input);
525 0 : FloatRegister output = ToFloatRegister(lir->output());
526 :
527 0 : Label isDouble, isInt32, isBool, isNull, isUndefined, done;
528 0 : bool hasBoolean = false, hasNull = false, hasUndefined = false;
529 :
530 : {
531 0 : ScratchTagScope tag(masm, operand);
532 0 : masm.splitTagForTest(operand, tag);
533 :
534 0 : masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
535 0 : masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
536 :
537 0 : if (mir->conversion() != MToFPInstruction::NumbersOnly) {
538 0 : masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
539 0 : masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
540 0 : hasBoolean = true;
541 0 : hasUndefined = true;
542 0 : if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) {
543 0 : masm.branchTestNull(Assembler::Equal, tag, &isNull);
544 0 : hasNull = true;
545 : }
546 : }
547 : }
548 :
549 0 : bailout(lir->snapshot());
550 :
551 0 : if (hasNull) {
552 0 : masm.bind(&isNull);
553 0 : masm.loadConstantFloat32(0.0f, output);
554 0 : masm.jump(&done);
555 : }
556 :
557 0 : if (hasUndefined) {
558 0 : masm.bind(&isUndefined);
559 0 : masm.loadConstantFloat32(float(GenericNaN()), output);
560 0 : masm.jump(&done);
561 : }
562 :
563 0 : if (hasBoolean) {
564 0 : masm.bind(&isBool);
565 0 : masm.boolValueToFloat32(operand, output);
566 0 : masm.jump(&done);
567 : }
568 :
569 0 : masm.bind(&isInt32);
570 0 : masm.int32ValueToFloat32(operand, output);
571 0 : masm.jump(&done);
572 :
573 0 : masm.bind(&isDouble);
574 : // ARM and MIPS may not have a double register available if we've
575 : // allocated output as a float32.
576 : #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32)
577 : masm.unboxDouble(operand, ScratchDoubleReg);
578 : masm.convertDoubleToFloat32(ScratchDoubleReg, output);
579 : #else
580 0 : masm.unboxDouble(operand, output);
581 0 : masm.convertDoubleToFloat32(output, output);
582 : #endif
583 0 : masm.bind(&done);
584 0 : }
585 :
586 : void
587 4 : CodeGenerator::visitInt32ToDouble(LInt32ToDouble* lir)
588 : {
589 0 : masm.convertInt32ToDouble(ToRegister(lir->input()), ToFloatRegister(lir->output()));
590 4 : }
591 :
592 : void
593 0 : CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble* lir)
594 : {
595 0 : masm.convertFloat32ToDouble(ToFloatRegister(lir->input()), ToFloatRegister(lir->output()));
596 0 : }
597 :
598 : void
599 0 : CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32* lir)
600 : {
601 0 : masm.convertDoubleToFloat32(ToFloatRegister(lir->input()), ToFloatRegister(lir->output()));
602 0 : }
603 :
604 : void
605 0 : CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32* lir)
606 : {
607 0 : masm.convertInt32ToFloat32(ToRegister(lir->input()), ToFloatRegister(lir->output()));
608 0 : }
609 :
610 : void
611 0 : CodeGenerator::visitDoubleToInt32(LDoubleToInt32* lir)
612 : {
613 0 : Label fail;
614 0 : FloatRegister input = ToFloatRegister(lir->input());
615 0 : Register output = ToRegister(lir->output());
616 0 : masm.convertDoubleToInt32(input, output, &fail, lir->mir()->canBeNegativeZero());
617 0 : bailoutFrom(&fail, lir->snapshot());
618 0 : }
619 :
620 : void
621 0 : CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32* lir)
622 : {
623 0 : Label fail;
624 0 : FloatRegister input = ToFloatRegister(lir->input());
625 0 : Register output = ToRegister(lir->output());
626 0 : masm.convertFloat32ToInt32(input, output, &fail, lir->mir()->canBeNegativeZero());
627 0 : bailoutFrom(&fail, lir->snapshot());
628 0 : }
629 :
630 : void
631 11 : CodeGenerator::emitOOLTestObject(Register objreg,
632 : Label* ifEmulatesUndefined,
633 : Label* ifDoesntEmulateUndefined,
634 : Register scratch)
635 : {
636 0 : saveVolatile(scratch);
637 0 : masm.setupUnalignedABICall(scratch);
638 0 : masm.passABIArg(objreg);
639 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::EmulatesUndefined));
640 0 : masm.storeCallBoolResult(scratch);
641 22 : restoreVolatile(scratch);
642 :
643 0 : masm.branchIfTrueBool(scratch, ifEmulatesUndefined);
644 0 : masm.jump(ifDoesntEmulateUndefined);
645 11 : }
646 :
647 : // Base out-of-line code generator for all tests of the truthiness of an
648 : // object, where the object might not be truthy. (Recall that per spec all
649 : // objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class
650 : // flag to permit objects to look like |undefined| in certain contexts,
651 : // including in object truthiness testing.) We check truthiness inline except
652 : // when we're testing it on a proxy (or if TI guarantees us that the specified
653 : // object will never emulate |undefined|), in which case out-of-line code will
654 : // call EmulatesUndefined for a conclusive answer.
655 : class OutOfLineTestObject : public OutOfLineCodeBase<CodeGenerator>
656 : {
657 : Register objreg_;
658 : Register scratch_;
659 :
660 : Label* ifEmulatesUndefined_;
661 : Label* ifDoesntEmulateUndefined_;
662 :
663 : #ifdef DEBUG
664 11 : bool initialized() { return ifEmulatesUndefined_ != nullptr; }
665 : #endif
666 :
667 : public:
668 : OutOfLineTestObject()
669 : #ifdef DEBUG
670 22 : : ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr)
671 : #endif
672 : { }
673 :
674 0 : void accept(CodeGenerator* codegen) final {
675 0 : MOZ_ASSERT(initialized());
676 0 : codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_, ifDoesntEmulateUndefined_,
677 0 : scratch_);
678 11 : }
679 :
680 : // Specify the register where the object to be tested is found, labels to
681 : // jump to if the object is truthy or falsy, and a scratch register for
682 : // use in the out-of-line path.
683 11 : void setInputAndTargets(Register objreg, Label* ifEmulatesUndefined, Label* ifDoesntEmulateUndefined,
684 : Register scratch)
685 : {
686 0 : MOZ_ASSERT(!initialized());
687 0 : MOZ_ASSERT(ifEmulatesUndefined);
688 0 : objreg_ = objreg;
689 0 : scratch_ = scratch;
690 0 : ifEmulatesUndefined_ = ifEmulatesUndefined;
691 0 : ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined;
692 11 : }
693 : };
694 :
695 : // A subclass of OutOfLineTestObject containing two extra labels, for use when
696 : // the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line
697 : // code. The user should bind these labels in inline code, and specify them as
698 : // targets via setInputAndTargets, as appropriate.
699 : class OutOfLineTestObjectWithLabels : public OutOfLineTestObject
700 : {
701 : Label label1_;
702 : Label label2_;
703 :
704 : public:
705 0 : OutOfLineTestObjectWithLabels() { }
706 :
707 0 : Label* label1() { return &label1_; }
708 0 : Label* label2() { return &label2_; }
709 : };
710 :
711 : void
712 11 : CodeGenerator::testObjectEmulatesUndefinedKernel(Register objreg,
713 : Label* ifEmulatesUndefined,
714 : Label* ifDoesntEmulateUndefined,
715 : Register scratch, OutOfLineTestObject* ool)
716 : {
717 11 : ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined, scratch);
718 :
719 : // Perform a fast-path check of the object's class flags if the object's
720 : // not a proxy. Let out-of-line code handle the slow cases that require
721 : // saving registers, making a function call, and restoring registers.
722 0 : masm.branchIfObjectEmulatesUndefined(objreg, scratch, ool->entry(), ifEmulatesUndefined);
723 11 : }
724 :
725 : void
726 0 : CodeGenerator::branchTestObjectEmulatesUndefined(Register objreg,
727 : Label* ifEmulatesUndefined,
728 : Label* ifDoesntEmulateUndefined,
729 : Register scratch, OutOfLineTestObject* ool)
730 : {
731 0 : MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(),
732 : "ifDoesntEmulateUndefined will be bound to the fallthrough path");
733 :
734 : testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
735 0 : scratch, ool);
736 0 : masm.bind(ifDoesntEmulateUndefined);
737 0 : }
738 :
739 : void
740 0 : CodeGenerator::testObjectEmulatesUndefined(Register objreg,
741 : Label* ifEmulatesUndefined,
742 : Label* ifDoesntEmulateUndefined,
743 : Register scratch, OutOfLineTestObject* ool)
744 : {
745 : testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
746 0 : scratch, ool);
747 1 : masm.jump(ifDoesntEmulateUndefined);
748 0 : }
749 :
750 : void
751 13 : CodeGenerator::testValueTruthyKernel(const ValueOperand& value,
752 : const LDefinition* scratch1, const LDefinition* scratch2,
753 : FloatRegister fr,
754 : Label* ifTruthy, Label* ifFalsy,
755 : OutOfLineTestObject* ool,
756 : MDefinition* valueMIR)
757 : {
758 : // Count the number of possible type tags we might have, so we'll know when
759 : // we've checked them all and hence can avoid emitting a tag check for the
760 : // last one. In particular, whenever tagCount is 1 that means we've tried
761 : // all but one of them already so we know exactly what's left based on the
762 : // mightBe* booleans.
763 0 : bool mightBeUndefined = valueMIR->mightBeType(MIRType::Undefined);
764 0 : bool mightBeNull = valueMIR->mightBeType(MIRType::Null);
765 0 : bool mightBeBoolean = valueMIR->mightBeType(MIRType::Boolean);
766 0 : bool mightBeInt32 = valueMIR->mightBeType(MIRType::Int32);
767 0 : bool mightBeObject = valueMIR->mightBeType(MIRType::Object);
768 0 : bool mightBeString = valueMIR->mightBeType(MIRType::String);
769 0 : bool mightBeSymbol = valueMIR->mightBeType(MIRType::Symbol);
770 0 : bool mightBeDouble = valueMIR->mightBeType(MIRType::Double);
771 0 : int tagCount = int(mightBeUndefined) + int(mightBeNull) +
772 0 : int(mightBeBoolean) + int(mightBeInt32) + int(mightBeObject) +
773 13 : int(mightBeString) + int(mightBeSymbol) + int(mightBeDouble);
774 :
775 13 : MOZ_ASSERT_IF(!valueMIR->emptyResultTypeSet(), tagCount > 0);
776 :
777 : // If we know we're null or undefined, we're definitely falsy, no
778 : // need to even check the tag.
779 0 : if (int(mightBeNull) + int(mightBeUndefined) == tagCount) {
780 0 : masm.jump(ifFalsy);
781 4 : return;
782 : }
783 :
784 0 : ScratchTagScope tag(masm, value);
785 18 : masm.splitTagForTest(value, tag);
786 :
787 0 : if (mightBeUndefined) {
788 0 : MOZ_ASSERT(tagCount > 1);
789 0 : masm.branchTestUndefined(Assembler::Equal, tag, ifFalsy);
790 7 : --tagCount;
791 : }
792 :
793 0 : if (mightBeNull) {
794 0 : MOZ_ASSERT(tagCount > 1);
795 0 : masm.branchTestNull(Assembler::Equal, tag, ifFalsy);
796 2 : --tagCount;
797 : }
798 :
799 0 : if (mightBeBoolean) {
800 0 : MOZ_ASSERT(tagCount != 0);
801 0 : Label notBoolean;
802 0 : if (tagCount != 1)
803 2 : masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean);
804 : {
805 0 : ScratchTagScopeRelease _(&tag);
806 3 : masm.branchTestBooleanTruthy(false, value, ifFalsy);
807 : }
808 0 : if (tagCount != 1)
809 2 : masm.jump(ifTruthy);
810 : // Else just fall through to truthiness.
811 0 : masm.bind(¬Boolean);
812 3 : --tagCount;
813 : }
814 :
815 0 : if (mightBeInt32) {
816 0 : MOZ_ASSERT(tagCount != 0);
817 0 : Label notInt32;
818 0 : if (tagCount != 1)
819 1 : masm.branchTestInt32(Assembler::NotEqual, tag, ¬Int32);
820 : {
821 0 : ScratchTagScopeRelease _(&tag);
822 1 : masm.branchTestInt32Truthy(false, value, ifFalsy);
823 : }
824 0 : if (tagCount != 1)
825 1 : masm.jump(ifTruthy);
826 : // Else just fall through to truthiness.
827 0 : masm.bind(¬Int32);
828 1 : --tagCount;
829 : }
830 :
831 0 : if (mightBeObject) {
832 0 : MOZ_ASSERT(tagCount != 0);
833 0 : if (ool) {
834 2 : Label notObject;
835 :
836 0 : if (tagCount != 1)
837 1 : masm.branchTestObject(Assembler::NotEqual, tag, ¬Object);
838 :
839 : {
840 0 : ScratchTagScopeRelease _(&tag);
841 0 : Register objreg = masm.extractObject(value, ToRegister(scratch1));
842 2 : testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy, ToRegister(scratch2), ool);
843 : }
844 :
845 1 : masm.bind(¬Object);
846 : } else {
847 0 : if (tagCount != 1)
848 0 : masm.branchTestObject(Assembler::Equal, tag, ifTruthy);
849 : // Else just fall through to truthiness.
850 : }
851 4 : --tagCount;
852 : } else {
853 5 : MOZ_ASSERT(!ool,
854 : "We better not have an unused OOL path, since the code generator will try to "
855 : "generate code for it but we never set up its labels, which will cause null "
856 : "derefs of those labels.");
857 : }
858 :
859 9 : if (mightBeString) {
860 : // Test if a string is non-empty.
861 0 : MOZ_ASSERT(tagCount != 0);
862 0 : Label notString;
863 0 : if (tagCount != 1)
864 0 : masm.branchTestString(Assembler::NotEqual, tag, ¬String);
865 : {
866 0 : ScratchTagScopeRelease _(&tag);
867 10 : masm.branchTestStringTruthy(false, value, ifFalsy);
868 : }
869 0 : if (tagCount != 1)
870 0 : masm.jump(ifTruthy);
871 : // Else just fall through to truthiness.
872 0 : masm.bind(¬String);
873 5 : --tagCount;
874 : }
875 :
876 9 : if (mightBeSymbol) {
877 : // All symbols are truthy.
878 0 : MOZ_ASSERT(tagCount != 0);
879 0 : if (tagCount != 1)
880 0 : masm.branchTestSymbol(Assembler::Equal, tag, ifTruthy);
881 : // Else fall through to ifTruthy.
882 0 : --tagCount;
883 : }
884 :
885 0 : if (mightBeDouble) {
886 0 : MOZ_ASSERT(tagCount == 1);
887 : // If we reach here the value is a double.
888 : {
889 0 : ScratchTagScopeRelease _(&tag);
890 0 : masm.unboxDouble(value, fr);
891 0 : masm.branchTestDoubleTruthy(false, fr, ifFalsy);
892 : }
893 0 : --tagCount;
894 : }
895 :
896 9 : MOZ_ASSERT(tagCount == 0);
897 :
898 : // Fall through for truthy.
899 : }
900 :
901 : void
902 0 : CodeGenerator::testValueTruthy(const ValueOperand& value,
903 : const LDefinition* scratch1, const LDefinition* scratch2,
904 : FloatRegister fr,
905 : Label* ifTruthy, Label* ifFalsy,
906 : OutOfLineTestObject* ool,
907 : MDefinition* valueMIR)
908 : {
909 0 : testValueTruthyKernel(value, scratch1, scratch2, fr, ifTruthy, ifFalsy, ool, valueMIR);
910 1 : masm.jump(ifTruthy);
911 0 : }
912 :
913 : void
914 10 : CodeGenerator::visitTestOAndBranch(LTestOAndBranch* lir)
915 : {
916 0 : MIRType inputType = lir->mir()->input()->type();
917 20 : MOZ_ASSERT(inputType == MIRType::ObjectOrNull || lir->mir()->operandMightEmulateUndefined(),
918 : "If the object couldn't emulate undefined, this should have been folded.");
919 :
920 0 : Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
921 0 : Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
922 20 : Register input = ToRegister(lir->input());
923 :
924 0 : if (lir->mir()->operandMightEmulateUndefined()) {
925 0 : if (inputType == MIRType::ObjectOrNull)
926 0 : masm.branchTestPtr(Assembler::Zero, input, input, falsy);
927 :
928 0 : OutOfLineTestObject* ool = new(alloc()) OutOfLineTestObject();
929 20 : addOutOfLineCode(ool, lir->mir());
930 :
931 20 : testObjectEmulatesUndefined(input, falsy, truthy, ToRegister(lir->temp()), ool);
932 : } else {
933 0 : MOZ_ASSERT(inputType == MIRType::ObjectOrNull);
934 0 : testZeroEmitBranch(Assembler::NotEqual, input, lir->ifTruthy(), lir->ifFalsy());
935 : }
936 10 : }
937 :
938 : void
939 13 : CodeGenerator::visitTestVAndBranch(LTestVAndBranch* lir)
940 : {
941 0 : OutOfLineTestObject* ool = nullptr;
942 39 : MDefinition* input = lir->mir()->input();
943 : // Unfortunately, it's possible that someone (e.g. phi elimination) switched
944 : // out our input after we did cacheOperandMightEmulateUndefined. So we
945 : // might think it can emulate undefined _and_ know that it can't be an
946 : // object.
947 0 : if (lir->mir()->operandMightEmulateUndefined() && input->mightBeType(MIRType::Object)) {
948 0 : ool = new(alloc()) OutOfLineTestObject();
949 2 : addOutOfLineCode(ool, lir->mir());
950 : }
951 :
952 0 : Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
953 13 : Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
954 :
955 65 : testValueTruthy(ToValue(lir, LTestVAndBranch::Input),
956 : lir->temp1(), lir->temp2(),
957 : ToFloatRegister(lir->tempFloat()),
958 0 : truthy, falsy, ool, input);
959 13 : }
960 :
961 : void
962 8 : CodeGenerator::visitFunctionDispatch(LFunctionDispatch* lir)
963 : {
964 0 : MFunctionDispatch* mir = lir->mir();
965 16 : Register input = ToRegister(lir->input());
966 : Label* lastLabel;
967 : size_t casesWithFallback;
968 :
969 : // Determine if the last case is fallback or an ordinary case.
970 0 : if (!mir->hasFallback()) {
971 0 : MOZ_ASSERT(mir->numCases() > 0);
972 0 : casesWithFallback = mir->numCases();
973 18 : lastLabel = skipTrivialBlocks(mir->getCaseBlock(mir->numCases() - 1))->lir()->label();
974 : } else {
975 0 : casesWithFallback = mir->numCases() + 1;
976 2 : lastLabel = skipTrivialBlocks(mir->getFallback())->lir()->label();
977 : }
978 :
979 : // Compare function pointers, except for the last case.
980 0 : for (size_t i = 0; i < casesWithFallback - 1; i++) {
981 0 : MOZ_ASSERT(i < mir->numCases());
982 0 : LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir();
983 0 : if (ObjectGroup* funcGroup = mir->getCaseObjectGroup(i)) {
984 0 : masm.branchTestObjGroupUnsafe(Assembler::Equal, input, funcGroup, target->label());
985 : } else {
986 0 : JSFunction* func = mir->getCase(i);
987 20 : masm.branchPtr(Assembler::Equal, input, ImmGCPtr(func), target->label());
988 : }
989 : }
990 :
991 : // Jump to the last case.
992 0 : masm.jump(lastLabel);
993 8 : }
994 :
995 : void
996 0 : CodeGenerator::visitObjectGroupDispatch(LObjectGroupDispatch* lir)
997 : {
998 0 : MObjectGroupDispatch* mir = lir->mir();
999 0 : Register input = ToRegister(lir->input());
1000 0 : Register temp = ToRegister(lir->temp());
1001 :
1002 : // Load the incoming ObjectGroup in temp.
1003 0 : masm.loadObjGroupUnsafe(input, temp);
1004 :
1005 : // Compare ObjectGroups.
1006 0 : MacroAssembler::BranchGCPtr lastBranch;
1007 0 : LBlock* lastBlock = nullptr;
1008 0 : InlinePropertyTable* propTable = mir->propTable();
1009 0 : for (size_t i = 0; i < mir->numCases(); i++) {
1010 0 : JSFunction* func = mir->getCase(i);
1011 0 : LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir();
1012 :
1013 0 : DebugOnly<bool> found = false;
1014 0 : for (size_t j = 0; j < propTable->numEntries(); j++) {
1015 0 : if (propTable->getFunction(j) != func)
1016 : continue;
1017 :
1018 0 : if (lastBranch.isInitialized())
1019 0 : lastBranch.emit(masm);
1020 :
1021 0 : ObjectGroup* group = propTable->getObjectGroup(j);
1022 0 : lastBranch = MacroAssembler::BranchGCPtr(Assembler::Equal, temp, ImmGCPtr(group),
1023 : target->label());
1024 0 : lastBlock = target;
1025 0 : found = true;
1026 : }
1027 0 : MOZ_ASSERT(found);
1028 : }
1029 :
1030 : // Jump to fallback block if we have an unknown ObjectGroup. If there's no
1031 : // fallback block, we should have handled all cases.
1032 :
1033 0 : if (!mir->hasFallback()) {
1034 0 : MOZ_ASSERT(lastBranch.isInitialized());
1035 : #ifdef DEBUG
1036 0 : Label ok;
1037 0 : lastBranch.relink(&ok);
1038 0 : lastBranch.emit(masm);
1039 0 : masm.assumeUnreachable("Unexpected ObjectGroup");
1040 0 : masm.bind(&ok);
1041 : #endif
1042 0 : if (!isNextBlock(lastBlock))
1043 0 : masm.jump(lastBlock->label());
1044 : return;
1045 : }
1046 :
1047 0 : LBlock* fallback = skipTrivialBlocks(mir->getFallback())->lir();
1048 0 : if (!lastBranch.isInitialized()) {
1049 0 : if (!isNextBlock(fallback))
1050 0 : masm.jump(fallback->label());
1051 : return;
1052 : }
1053 :
1054 0 : lastBranch.invertCondition();
1055 0 : lastBranch.relink(fallback->label());
1056 0 : lastBranch.emit(masm);
1057 :
1058 0 : if (!isNextBlock(lastBlock))
1059 0 : masm.jump(lastBlock->label());
1060 : }
1061 :
1062 : void
1063 0 : CodeGenerator::visitBooleanToString(LBooleanToString* lir)
1064 : {
1065 0 : Register input = ToRegister(lir->input());
1066 0 : Register output = ToRegister(lir->output());
1067 0 : const JSAtomState& names = gen->runtime->names();
1068 0 : Label true_, done;
1069 :
1070 0 : masm.branchTest32(Assembler::NonZero, input, input, &true_);
1071 0 : masm.movePtr(ImmGCPtr(names.false_), output);
1072 0 : masm.jump(&done);
1073 :
1074 0 : masm.bind(&true_);
1075 0 : masm.movePtr(ImmGCPtr(names.true_), output);
1076 :
1077 0 : masm.bind(&done);
1078 0 : }
1079 :
1080 : void
1081 45 : CodeGenerator::emitIntToString(Register input, Register output, Label* ool)
1082 : {
1083 45 : masm.boundsCheck32PowerOfTwo(input, StaticStrings::INT_STATIC_LIMIT, ool);
1084 :
1085 : // Fast path for small integers.
1086 0 : masm.movePtr(ImmPtr(&gen->runtime->staticStrings().intStaticTable), output);
1087 0 : masm.loadPtr(BaseIndex(output, input, ScalePointer), output);
1088 45 : }
1089 :
1090 : typedef JSFlatString* (*IntToStringFn)(JSContext*, int);
1091 0 : static const VMFunction IntToStringInfo =
1092 3 : FunctionInfo<IntToStringFn>(Int32ToString<CanGC>, "Int32ToString");
1093 :
1094 : void
1095 45 : CodeGenerator::visitIntToString(LIntToString* lir)
1096 : {
1097 0 : Register input = ToRegister(lir->input());
1098 90 : Register output = ToRegister(lir->output());
1099 :
1100 0 : OutOfLineCode* ool = oolCallVM(IntToStringInfo, lir, ArgList(input),
1101 135 : StoreRegisterTo(output));
1102 :
1103 45 : emitIntToString(input, output, ool->entry());
1104 :
1105 0 : masm.bind(ool->rejoin());
1106 45 : }
1107 :
1108 : typedef JSString* (*DoubleToStringFn)(JSContext*, double);
1109 0 : static const VMFunction DoubleToStringInfo =
1110 3 : FunctionInfo<DoubleToStringFn>(NumberToString<CanGC>, "NumberToString");
1111 :
1112 : void
1113 0 : CodeGenerator::visitDoubleToString(LDoubleToString* lir)
1114 : {
1115 0 : FloatRegister input = ToFloatRegister(lir->input());
1116 0 : Register temp = ToRegister(lir->tempInt());
1117 0 : Register output = ToRegister(lir->output());
1118 :
1119 0 : OutOfLineCode* ool = oolCallVM(DoubleToStringInfo, lir, ArgList(input),
1120 0 : StoreRegisterTo(output));
1121 :
1122 : // Try double to integer conversion and run integer to string code.
1123 0 : masm.convertDoubleToInt32(input, temp, ool->entry(), true);
1124 0 : emitIntToString(temp, output, ool->entry());
1125 :
1126 0 : masm.bind(ool->rejoin());
1127 0 : }
1128 :
1129 : typedef JSString* (*PrimitiveToStringFn)(JSContext*, HandleValue);
1130 0 : static const VMFunction PrimitiveToStringInfo =
1131 3 : FunctionInfo<PrimitiveToStringFn>(ToStringSlow, "ToStringSlow");
1132 :
1133 : void
1134 0 : CodeGenerator::visitValueToString(LValueToString* lir)
1135 : {
1136 0 : ValueOperand input = ToValue(lir, LValueToString::Input);
1137 0 : Register output = ToRegister(lir->output());
1138 :
1139 0 : OutOfLineCode* ool = oolCallVM(PrimitiveToStringInfo, lir, ArgList(input),
1140 0 : StoreRegisterTo(output));
1141 :
1142 0 : Label done;
1143 0 : Register tag = masm.extractTag(input, output);
1144 0 : const JSAtomState& names = gen->runtime->names();
1145 :
1146 : // String
1147 0 : if (lir->mir()->input()->mightBeType(MIRType::String)) {
1148 0 : Label notString;
1149 0 : masm.branchTestString(Assembler::NotEqual, tag, ¬String);
1150 0 : masm.unboxString(input, output);
1151 0 : masm.jump(&done);
1152 0 : masm.bind(¬String);
1153 : }
1154 :
1155 : // Integer
1156 0 : if (lir->mir()->input()->mightBeType(MIRType::Int32)) {
1157 0 : Label notInteger;
1158 0 : masm.branchTestInt32(Assembler::NotEqual, tag, ¬Integer);
1159 0 : Register unboxed = ToTempUnboxRegister(lir->tempToUnbox());
1160 0 : unboxed = masm.extractInt32(input, unboxed);
1161 0 : emitIntToString(unboxed, output, ool->entry());
1162 0 : masm.jump(&done);
1163 0 : masm.bind(¬Integer);
1164 : }
1165 :
1166 : // Double
1167 0 : if (lir->mir()->input()->mightBeType(MIRType::Double)) {
1168 : // Note: no fastpath. Need two extra registers and can only convert doubles
1169 : // that fit integers and are smaller than StaticStrings::INT_STATIC_LIMIT.
1170 0 : masm.branchTestDouble(Assembler::Equal, tag, ool->entry());
1171 : }
1172 :
1173 : // Undefined
1174 0 : if (lir->mir()->input()->mightBeType(MIRType::Undefined)) {
1175 0 : Label notUndefined;
1176 0 : masm.branchTestUndefined(Assembler::NotEqual, tag, ¬Undefined);
1177 0 : masm.movePtr(ImmGCPtr(names.undefined), output);
1178 0 : masm.jump(&done);
1179 0 : masm.bind(¬Undefined);
1180 : }
1181 :
1182 : // Null
1183 0 : if (lir->mir()->input()->mightBeType(MIRType::Null)) {
1184 0 : Label notNull;
1185 0 : masm.branchTestNull(Assembler::NotEqual, tag, ¬Null);
1186 0 : masm.movePtr(ImmGCPtr(names.null), output);
1187 0 : masm.jump(&done);
1188 0 : masm.bind(¬Null);
1189 : }
1190 :
1191 : // Boolean
1192 0 : if (lir->mir()->input()->mightBeType(MIRType::Boolean)) {
1193 0 : Label notBoolean, true_;
1194 0 : masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean);
1195 0 : masm.branchTestBooleanTruthy(true, input, &true_);
1196 0 : masm.movePtr(ImmGCPtr(names.false_), output);
1197 0 : masm.jump(&done);
1198 0 : masm.bind(&true_);
1199 0 : masm.movePtr(ImmGCPtr(names.true_), output);
1200 0 : masm.jump(&done);
1201 0 : masm.bind(¬Boolean);
1202 : }
1203 :
1204 : // Object
1205 0 : if (lir->mir()->input()->mightBeType(MIRType::Object)) {
1206 : // Bail.
1207 0 : MOZ_ASSERT(lir->mir()->fallible());
1208 0 : Label bail;
1209 0 : masm.branchTestObject(Assembler::Equal, tag, &bail);
1210 0 : bailoutFrom(&bail, lir->snapshot());
1211 : }
1212 :
1213 : // Symbol
1214 0 : if (lir->mir()->input()->mightBeType(MIRType::Symbol)) {
1215 : // Bail.
1216 0 : MOZ_ASSERT(lir->mir()->fallible());
1217 0 : Label bail;
1218 0 : masm.branchTestSymbol(Assembler::Equal, tag, &bail);
1219 0 : bailoutFrom(&bail, lir->snapshot());
1220 : }
1221 :
1222 : #ifdef DEBUG
1223 0 : masm.assumeUnreachable("Unexpected type for MValueToString.");
1224 : #endif
1225 :
1226 0 : masm.bind(&done);
1227 0 : masm.bind(ool->rejoin());
1228 0 : }
1229 :
1230 : typedef JSObject* (*ToObjectFn)(JSContext*, HandleValue, bool);
1231 0 : static const VMFunction ToObjectInfo =
1232 3 : FunctionInfo<ToObjectFn>(ToObjectSlow, "ToObjectSlow");
1233 :
1234 : void
1235 0 : CodeGenerator::visitValueToObject(LValueToObject* lir)
1236 : {
1237 0 : ValueOperand input = ToValue(lir, LValueToObject::Input);
1238 0 : Register output = ToRegister(lir->output());
1239 :
1240 0 : OutOfLineCode* ool = oolCallVM(ToObjectInfo, lir, ArgList(input, Imm32(0)),
1241 0 : StoreRegisterTo(output));
1242 :
1243 0 : masm.branchTestObject(Assembler::NotEqual, input, ool->entry());
1244 0 : masm.unboxObject(input, output);
1245 :
1246 0 : masm.bind(ool->rejoin());
1247 0 : }
1248 :
1249 : void
1250 0 : CodeGenerator::visitValueToObjectOrNull(LValueToObjectOrNull* lir)
1251 : {
1252 0 : ValueOperand input = ToValue(lir, LValueToObjectOrNull::Input);
1253 0 : Register output = ToRegister(lir->output());
1254 :
1255 0 : OutOfLineCode* ool = oolCallVM(ToObjectInfo, lir, ArgList(input, Imm32(0)),
1256 0 : StoreRegisterTo(output));
1257 :
1258 0 : Label isObject;
1259 0 : masm.branchTestObject(Assembler::Equal, input, &isObject);
1260 0 : masm.branchTestNull(Assembler::NotEqual, input, ool->entry());
1261 :
1262 0 : masm.movePtr(ImmWord(0), output);
1263 0 : masm.jump(ool->rejoin());
1264 :
1265 0 : masm.bind(&isObject);
1266 0 : masm.unboxObject(input, output);
1267 :
1268 0 : masm.bind(ool->rejoin());
1269 0 : }
1270 :
1271 : static void
1272 0 : EmitStoreBufferMutation(MacroAssembler& masm, Register holder, size_t offset,
1273 : Register buffer,
1274 : LiveGeneralRegisterSet& liveVolatiles,
1275 : void (*fun)(js::gc::StoreBuffer*, js::gc::Cell**))
1276 : {
1277 0 : Label callVM;
1278 0 : Label exit;
1279 :
1280 : // Call into the VM to barrier the write. The only registers that need to
1281 : // be preserved are those in liveVolatiles, so once they are saved on the
1282 : // stack all volatile registers are available for use.
1283 0 : masm.bind(&callVM);
1284 0 : masm.PushRegsInMask(liveVolatiles);
1285 :
1286 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
1287 0 : regs.takeUnchecked(buffer);
1288 0 : regs.takeUnchecked(holder);
1289 0 : Register addrReg = regs.takeAny();
1290 :
1291 0 : masm.computeEffectiveAddress(Address(holder, offset), addrReg);
1292 :
1293 0 : bool needExtraReg = !regs.hasAny<GeneralRegisterSet::DefaultType>();
1294 0 : if (needExtraReg) {
1295 0 : masm.push(holder);
1296 0 : masm.setupUnalignedABICall(holder);
1297 : } else {
1298 0 : masm.setupUnalignedABICall(regs.takeAny());
1299 : }
1300 0 : masm.passABIArg(buffer);
1301 0 : masm.passABIArg(addrReg);
1302 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, fun), MoveOp::GENERAL,
1303 0 : CheckUnsafeCallWithABI::DontCheckOther);
1304 :
1305 0 : if (needExtraReg)
1306 0 : masm.pop(holder);
1307 0 : masm.PopRegsInMask(liveVolatiles);
1308 0 : masm.bind(&exit);
1309 0 : }
1310 :
1311 : // Warning: this function modifies prev and next.
1312 : static void
1313 0 : EmitPostWriteBarrierS(MacroAssembler& masm,
1314 : Register holder, size_t offset,
1315 : Register prev, Register next,
1316 : LiveGeneralRegisterSet& liveVolatiles)
1317 : {
1318 0 : Label exit;
1319 0 : Label checkRemove, putCell;
1320 :
1321 : // if (next && (buffer = next->storeBuffer()))
1322 : // but we never pass in nullptr for next.
1323 0 : Register storebuffer = next;
1324 0 : masm.loadStoreBuffer(next, storebuffer);
1325 0 : masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &checkRemove);
1326 :
1327 : // if (prev && prev->storeBuffer())
1328 0 : masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &putCell);
1329 0 : masm.loadStoreBuffer(prev, prev);
1330 0 : masm.branchPtr(Assembler::NotEqual, prev, ImmWord(0), &exit);
1331 :
1332 : // buffer->putCell(cellp)
1333 0 : masm.bind(&putCell);
1334 : EmitStoreBufferMutation(masm, holder, offset, storebuffer, liveVolatiles,
1335 0 : JSString::addCellAddressToStoreBuffer);
1336 0 : masm.jump(&exit);
1337 :
1338 : // if (prev && (buffer = prev->storeBuffer()))
1339 0 : masm.bind(&checkRemove);
1340 0 : masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &exit);
1341 0 : masm.loadStoreBuffer(prev, storebuffer);
1342 0 : masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &exit);
1343 : EmitStoreBufferMutation(masm, holder, offset, storebuffer, liveVolatiles,
1344 0 : JSString::removeCellAddressFromStoreBuffer);
1345 :
1346 0 : masm.bind(&exit);
1347 0 : }
1348 :
1349 : typedef JSObject* (*CloneRegExpObjectFn)(JSContext*, Handle<RegExpObject*>);
1350 0 : static const VMFunction CloneRegExpObjectInfo =
1351 3 : FunctionInfo<CloneRegExpObjectFn>(CloneRegExpObject, "CloneRegExpObject");
1352 :
1353 : void
1354 0 : CodeGenerator::visitRegExp(LRegExp* lir)
1355 : {
1356 0 : Register output = ToRegister(lir->output());
1357 0 : Register temp = ToRegister(lir->temp());
1358 0 : JSObject* source = lir->mir()->source();
1359 :
1360 0 : OutOfLineCode* ool = oolCallVM(CloneRegExpObjectInfo, lir, ArgList(ImmGCPtr(source)),
1361 0 : StoreRegisterTo(output));
1362 0 : if (lir->mir()->hasShared()) {
1363 0 : TemplateObject templateObject(source);
1364 0 : masm.createGCObject(output, temp, templateObject, gc::DefaultHeap, ool->entry());
1365 : } else {
1366 0 : masm.jump(ool->entry());
1367 : }
1368 0 : masm.bind(ool->rejoin());
1369 0 : }
1370 :
1371 : // Amount of space to reserve on the stack when executing RegExps inline.
1372 : static const size_t RegExpReservedStack = sizeof(irregexp::InputOutputData)
1373 : + sizeof(MatchPairs)
1374 : + RegExpObject::MaxPairCount * sizeof(MatchPair);
1375 :
1376 : static size_t
1377 : RegExpPairsVectorStartOffset(size_t inputOutputDataStartOffset)
1378 : {
1379 3 : return inputOutputDataStartOffset + sizeof(irregexp::InputOutputData) + sizeof(MatchPairs);
1380 : }
1381 :
1382 : static Address
1383 : RegExpPairCountAddress(MacroAssembler& masm, size_t inputOutputDataStartOffset)
1384 : {
1385 4 : return Address(masm.getStackPointer(), inputOutputDataStartOffset
1386 : + sizeof(irregexp::InputOutputData)
1387 3 : + MatchPairs::offsetOfPairCount());
1388 : }
1389 :
1390 : // Prepare an InputOutputData and optional MatchPairs which space has been
1391 : // allocated for on the stack, and try to execute a RegExp on a string input.
1392 : // If the RegExp was successfully executed and matched the input, fallthrough,
1393 : // otherwise jump to notFound or failure.
1394 : static bool
1395 3 : PrepareAndExecuteRegExp(JSContext* cx, MacroAssembler& masm, Register regexp, Register input,
1396 : Register lastIndex,
1397 : Register temp1, Register temp2, Register temp3,
1398 : size_t inputOutputDataStartOffset,
1399 : RegExpShared::CompilationMode mode,
1400 : bool stringsCanBeInNursery,
1401 : Label* notFound, Label* failure)
1402 : {
1403 0 : size_t matchPairsStartOffset = inputOutputDataStartOffset + sizeof(irregexp::InputOutputData);
1404 3 : size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
1405 :
1406 : Address inputStartAddress(masm.getStackPointer(),
1407 6 : inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, inputStart));
1408 : Address inputEndAddress(masm.getStackPointer(),
1409 6 : inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, inputEnd));
1410 : Address matchesPointerAddress(masm.getStackPointer(),
1411 6 : inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, matches));
1412 : Address startIndexAddress(masm.getStackPointer(),
1413 6 : inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, startIndex));
1414 : Address endIndexAddress(masm.getStackPointer(),
1415 6 : inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, endIndex));
1416 : Address matchResultAddress(masm.getStackPointer(),
1417 6 : inputOutputDataStartOffset + offsetof(irregexp::InputOutputData, result));
1418 :
1419 6 : Address pairCountAddress = RegExpPairCountAddress(masm, inputOutputDataStartOffset);
1420 : Address pairsPointerAddress(masm.getStackPointer(),
1421 6 : matchPairsStartOffset + MatchPairs::offsetOfPairs());
1422 :
1423 6 : Address pairsVectorAddress(masm.getStackPointer(), pairsVectorStartOffset);
1424 :
1425 0 : RegExpStatics* res = GlobalObject::getRegExpStatics(cx, cx->global());
1426 3 : if (!res)
1427 : return false;
1428 : #ifdef JS_USE_LINK_REGISTER
1429 : if (mode != RegExpShared::MatchOnly)
1430 : masm.pushReturnAddress();
1431 : #endif
1432 3 : if (mode == RegExpShared::Normal) {
1433 : // First, fill in a skeletal MatchPairs instance on the stack. This will be
1434 : // passed to the OOL stub in the caller if we aren't able to execute the
1435 : // RegExp inline, and that stub needs to be able to determine whether the
1436 : // execution finished successfully.
1437 0 : masm.store32(Imm32(1), pairCountAddress);
1438 0 : masm.store32(Imm32(-1), pairsVectorAddress);
1439 0 : masm.computeEffectiveAddress(pairsVectorAddress, temp1);
1440 2 : masm.storePtr(temp1, pairsPointerAddress);
1441 : }
1442 :
1443 : // Check for a linear input string.
1444 3 : masm.branchIfRopeOrExternal(input, temp1, failure);
1445 :
1446 : // Get the RegExpShared for the RegExp.
1447 0 : masm.loadPtr(Address(regexp, NativeObject::getFixedSlotOffset(RegExpObject::PRIVATE_SLOT)), temp1);
1448 6 : masm.branchPtr(Assembler::Equal, temp1, ImmWord(0), failure);
1449 :
1450 : // ES6 21.2.2.2 step 2.
1451 : // See RegExp.cpp ExecuteRegExp for more detail.
1452 : {
1453 6 : Label done;
1454 :
1455 0 : masm.branchTest32(Assembler::Zero, Address(temp1, RegExpShared::offsetOfFlags()),
1456 3 : Imm32(UnicodeFlag), &done);
1457 :
1458 : // If input is latin1, there should not be surrogate pair.
1459 3 : masm.branchLatin1String(input, &done);
1460 :
1461 : // Check if |lastIndex > 0 && lastIndex < input->length()|.
1462 : // lastIndex should already have no sign here.
1463 0 : masm.branchTest32(Assembler::Zero, lastIndex, lastIndex, &done);
1464 0 : masm.loadStringLength(input, temp2);
1465 3 : masm.branch32(Assembler::AboveOrEqual, lastIndex, temp2, &done);
1466 :
1467 : // Check if input[lastIndex] is trail surrogate.
1468 0 : masm.loadStringChars(input, temp2, CharEncoding::TwoByte);
1469 0 : masm.computeEffectiveAddress(BaseIndex(temp2, lastIndex, TimesTwo), temp3);
1470 3 : masm.load16ZeroExtend(Address(temp3, 0), temp3);
1471 :
1472 0 : masm.branch32(Assembler::Below, temp3, Imm32(unicode::TrailSurrogateMin), &done);
1473 6 : masm.branch32(Assembler::Above, temp3, Imm32(unicode::TrailSurrogateMax), &done);
1474 :
1475 : // Check if input[lastIndex-1] is lead surrogate.
1476 0 : masm.move32(lastIndex, temp3);
1477 0 : masm.sub32(Imm32(1), temp3);
1478 0 : masm.computeEffectiveAddress(BaseIndex(temp2, temp3, TimesTwo), temp3);
1479 3 : masm.load16ZeroExtend(Address(temp3, 0), temp3);
1480 :
1481 0 : masm.branch32(Assembler::Below, temp3, Imm32(unicode::LeadSurrogateMin), &done);
1482 6 : masm.branch32(Assembler::Above, temp3, Imm32(unicode::LeadSurrogateMax), &done);
1483 :
1484 : // Move lastIndex to lead surrogate.
1485 3 : masm.subPtr(Imm32(1), lastIndex);
1486 :
1487 3 : masm.bind(&done);
1488 : }
1489 :
1490 3 : if (mode == RegExpShared::Normal) {
1491 : // Don't handle RegExps with excessive parens.
1492 0 : masm.load32(Address(temp1, RegExpShared::offsetOfParenCount()), temp2);
1493 4 : masm.branch32(Assembler::AboveOrEqual, temp2, Imm32(RegExpObject::MaxPairCount), failure);
1494 :
1495 : // Fill in the paren count in the MatchPairs on the stack.
1496 0 : masm.add32(Imm32(1), temp2);
1497 2 : masm.store32(temp2, pairCountAddress);
1498 : }
1499 :
1500 : // Load the code pointer for the type of input string we have, and compute
1501 : // the input start/end pointers in the InputOutputData.
1502 3 : Register codePointer = temp1;
1503 : {
1504 3 : masm.loadStringLength(input, temp3);
1505 :
1506 0 : Label isLatin1, done;
1507 3 : masm.branchLatin1String(input, &isLatin1);
1508 : {
1509 0 : masm.loadStringChars(input, temp2, CharEncoding::TwoByte);
1510 0 : masm.storePtr(temp2, inputStartAddress);
1511 0 : masm.lshiftPtr(Imm32(1), temp3);
1512 0 : masm.loadPtr(Address(temp1, RegExpShared::offsetOfTwoByteJitCode(mode)),
1513 0 : codePointer);
1514 6 : masm.jump(&done);
1515 : }
1516 3 : masm.bind(&isLatin1);
1517 : {
1518 0 : masm.loadStringChars(input, temp2, CharEncoding::Latin1);
1519 0 : masm.storePtr(temp2, inputStartAddress);
1520 0 : masm.loadPtr(Address(temp1, RegExpShared::offsetOfLatin1JitCode(mode)),
1521 3 : codePointer);
1522 : }
1523 3 : masm.bind(&done);
1524 :
1525 0 : masm.addPtr(temp3, temp2);
1526 3 : masm.storePtr(temp2, inputEndAddress);
1527 : }
1528 :
1529 : // Check the RegExpShared has been compiled for this type of input.
1530 0 : masm.branchPtr(Assembler::Equal, codePointer, ImmWord(0), failure);
1531 6 : masm.loadPtr(Address(codePointer, JitCode::offsetOfCode()), codePointer);
1532 :
1533 : // Finish filling in the InputOutputData instance on the stack.
1534 0 : if (mode == RegExpShared::Normal) {
1535 0 : masm.computeEffectiveAddress(Address(masm.getStackPointer(), matchPairsStartOffset), temp2);
1536 2 : masm.storePtr(temp2, matchesPointerAddress);
1537 : } else {
1538 : // Use InputOutputData.endIndex itself for output.
1539 0 : masm.computeEffectiveAddress(endIndexAddress, temp2);
1540 1 : masm.storePtr(temp2, endIndexAddress);
1541 : }
1542 0 : masm.storePtr(lastIndex, startIndexAddress);
1543 3 : masm.store32(Imm32(0), matchResultAddress);
1544 :
1545 : // Save any volatile inputs.
1546 0 : LiveGeneralRegisterSet volatileRegs;
1547 0 : if (lastIndex.volatile_())
1548 0 : volatileRegs.add(lastIndex);
1549 0 : if (input.volatile_())
1550 0 : volatileRegs.add(input);
1551 0 : if (regexp.volatile_())
1552 3 : volatileRegs.add(regexp);
1553 :
1554 : #ifdef JS_TRACE_LOGGING
1555 1 : if (TraceLogTextIdEnabled(TraceLogger_IrregexpExecute)) {
1556 0 : masm.push(temp1);
1557 0 : masm.loadTraceLogger(temp1);
1558 0 : masm.tracelogStartId(temp1, TraceLogger_IrregexpExecute);
1559 0 : masm.pop(temp1);
1560 : }
1561 : #endif
1562 :
1563 : // Execute the RegExp.
1564 0 : masm.computeEffectiveAddress(Address(masm.getStackPointer(), inputOutputDataStartOffset), temp2);
1565 0 : masm.PushRegsInMask(volatileRegs);
1566 0 : masm.setupUnalignedABICall(temp3);
1567 0 : masm.passABIArg(temp2);
1568 0 : masm.callWithABI(codePointer);
1569 3 : masm.PopRegsInMask(volatileRegs);
1570 :
1571 : #ifdef JS_TRACE_LOGGING
1572 1 : if (TraceLogTextIdEnabled(TraceLogger_IrregexpExecute)) {
1573 0 : masm.loadTraceLogger(temp1);
1574 0 : masm.tracelogStopId(temp1, TraceLogger_IrregexpExecute);
1575 : }
1576 : #endif
1577 :
1578 0 : Label success;
1579 0 : masm.branch32(Assembler::Equal, matchResultAddress,
1580 0 : Imm32(RegExpRunStatus_Success_NotFound), notFound);
1581 0 : masm.branch32(Assembler::Equal, matchResultAddress,
1582 3 : Imm32(RegExpRunStatus_Error), failure);
1583 :
1584 : // Lazily update the RegExpStatics.
1585 6 : masm.movePtr(ImmPtr(res), temp1);
1586 :
1587 0 : Address pendingInputAddress(temp1, RegExpStatics::offsetOfPendingInput());
1588 0 : Address matchesInputAddress(temp1, RegExpStatics::offsetOfMatchesInput());
1589 0 : Address lazySourceAddress(temp1, RegExpStatics::offsetOfLazySource());
1590 6 : Address lazyIndexAddress(temp1, RegExpStatics::offsetOfLazyIndex());
1591 :
1592 0 : masm.guardedCallPreBarrier(pendingInputAddress, MIRType::String);
1593 0 : masm.guardedCallPreBarrier(matchesInputAddress, MIRType::String);
1594 3 : masm.guardedCallPreBarrier(lazySourceAddress, MIRType::String);
1595 :
1596 3 : if (stringsCanBeInNursery) {
1597 : // Writing into RegExpStatics tenured memory; must post-barrier.
1598 0 : if (temp1.volatile_())
1599 0 : volatileRegs.add(temp1);
1600 :
1601 0 : masm.loadPtr(pendingInputAddress, temp2);
1602 0 : masm.storePtr(input, pendingInputAddress);
1603 0 : masm.movePtr(input, temp3);
1604 : EmitPostWriteBarrierS(masm, temp1, RegExpStatics::offsetOfPendingInput(),
1605 0 : temp2 /* prev */, temp3 /* next */, volatileRegs);
1606 :
1607 0 : masm.loadPtr(matchesInputAddress, temp2);
1608 0 : masm.storePtr(input, matchesInputAddress);
1609 0 : masm.movePtr(input, temp3);
1610 : EmitPostWriteBarrierS(masm, temp1, RegExpStatics::offsetOfMatchesInput(),
1611 0 : temp2 /* prev */, temp3 /* next */, volatileRegs);
1612 : } else {
1613 0 : masm.storePtr(input, pendingInputAddress);
1614 3 : masm.storePtr(input, matchesInputAddress);
1615 : }
1616 :
1617 0 : masm.storePtr(lastIndex, Address(temp1, RegExpStatics::offsetOfLazyIndex()));
1618 9 : masm.store32(Imm32(1), Address(temp1, RegExpStatics::offsetOfPendingLazyEvaluation()));
1619 :
1620 0 : masm.loadPtr(Address(regexp, NativeObject::getFixedSlotOffset(RegExpObject::PRIVATE_SLOT)), temp2);
1621 0 : masm.loadPtr(Address(temp2, RegExpShared::offsetOfSource()), temp3);
1622 0 : masm.storePtr(temp3, lazySourceAddress);
1623 0 : masm.load32(Address(temp2, RegExpShared::offsetOfFlags()), temp3);
1624 6 : masm.store32(temp3, Address(temp1, RegExpStatics::offsetOfLazyFlags()));
1625 :
1626 3 : if (mode == RegExpShared::MatchOnly) {
1627 : // endIndex is passed via temp3.
1628 1 : masm.load32(endIndexAddress, temp3);
1629 : }
1630 :
1631 : return true;
1632 : }
1633 :
1634 : static void
1635 : CopyStringChars(MacroAssembler& masm, Register to, Register from, Register len,
1636 : Register byteOpScratch, size_t fromWidth, size_t toWidth);
1637 :
1638 12 : class CreateDependentString
1639 : {
1640 : Register string_;
1641 : Register temp_;
1642 : Label* failure_;
1643 : enum class FallbackKind : uint8_t {
1644 : InlineString,
1645 : FatInlineString,
1646 : NotInlineString,
1647 : Count
1648 : };
1649 : mozilla::EnumeratedArray<FallbackKind, FallbackKind::Count, Label> fallbacks_, joins_;
1650 :
1651 : public:
1652 : // Generate code that creates DependentString.
1653 : // Caller should call generateFallback after masm.ret(), to generate
1654 : // fallback path.
1655 : void generate(MacroAssembler& masm, const JSAtomState& names,
1656 : CompileRuntime* runtime,
1657 : bool latin1, Register string,
1658 : Register base, Register temp1, Register temp2,
1659 : BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
1660 : bool stringsCanBeInNursery,
1661 : Label* failure);
1662 :
1663 : // Generate fallback path for creating DependentString.
1664 : void generateFallback(MacroAssembler& masm, LiveRegisterSet regsToSave);
1665 : };
1666 :
1667 : void
1668 2 : CreateDependentString::generate(MacroAssembler& masm, const JSAtomState& names,
1669 : CompileRuntime* runtime,
1670 : bool latin1, Register string,
1671 : Register base, Register temp1, Register temp2,
1672 : BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
1673 : bool stringsCanBeInNursery,
1674 : Label* failure)
1675 : {
1676 0 : string_ = string;
1677 0 : temp_ = temp2;
1678 2 : failure_ = failure;
1679 :
1680 : // Compute the string length.
1681 0 : masm.load32(startIndexAddress, temp2);
1682 0 : masm.load32(limitIndexAddress, temp1);
1683 2 : masm.sub32(temp2, temp1);
1684 :
1685 6 : Label done, nonEmpty;
1686 :
1687 : // Zero length matches use the empty string.
1688 0 : masm.branchTest32(Assembler::NonZero, temp1, temp1, &nonEmpty);
1689 0 : masm.movePtr(ImmGCPtr(names.empty), string);
1690 4 : masm.jump(&done);
1691 :
1692 2 : masm.bind(&nonEmpty);
1693 :
1694 4 : Label notInline;
1695 :
1696 : int32_t maxInlineLength = latin1
1697 0 : ? (int32_t) JSFatInlineString::MAX_LENGTH_LATIN1
1698 0 : : (int32_t) JSFatInlineString::MAX_LENGTH_TWO_BYTE;
1699 4 : masm.branch32(Assembler::Above, temp1, Imm32(maxInlineLength), ¬Inline);
1700 :
1701 : {
1702 : // Make a thin or fat inline string.
1703 6 : Label stringAllocated, fatInline;
1704 :
1705 : int32_t maxThinInlineLength = latin1
1706 0 : ? (int32_t) JSThinInlineString::MAX_LENGTH_LATIN1
1707 0 : : (int32_t) JSThinInlineString::MAX_LENGTH_TWO_BYTE;
1708 4 : masm.branch32(Assembler::Above, temp1, Imm32(maxThinInlineLength), &fatInline);
1709 :
1710 0 : int32_t thinFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_THIN_INLINE_FLAGS;
1711 0 : masm.newGCString(string, temp2, &fallbacks_[FallbackKind::InlineString], stringsCanBeInNursery);
1712 0 : masm.bind(&joins_[FallbackKind::InlineString]);
1713 0 : masm.store32(Imm32(thinFlags), Address(string, JSString::offsetOfFlags()));
1714 4 : masm.jump(&stringAllocated);
1715 :
1716 2 : masm.bind(&fatInline);
1717 :
1718 0 : int32_t fatFlags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::INIT_FAT_INLINE_FLAGS;
1719 0 : masm.newGCFatInlineString(string, temp2, &fallbacks_[FallbackKind::FatInlineString], stringsCanBeInNursery);
1720 0 : masm.bind(&joins_[FallbackKind::FatInlineString]);
1721 6 : masm.store32(Imm32(fatFlags), Address(string, JSString::offsetOfFlags()));
1722 :
1723 0 : masm.bind(&stringAllocated);
1724 4 : masm.store32(temp1, Address(string, JSString::offsetOfLength()));
1725 :
1726 0 : masm.push(string);
1727 4 : masm.push(base);
1728 :
1729 : // Adjust the start index address for the above pushes.
1730 0 : MOZ_ASSERT(startIndexAddress.base == masm.getStackPointer());
1731 0 : BaseIndex newStartIndexAddress = startIndexAddress;
1732 2 : newStartIndexAddress.offset += 2 * sizeof(void*);
1733 :
1734 : // Load chars pointer for the new string.
1735 2 : masm.loadInlineStringCharsForStore(string, string);
1736 :
1737 : // Load the source characters pointer.
1738 0 : masm.loadStringChars(base, temp2,
1739 0 : latin1 ? CharEncoding::Latin1 : CharEncoding::TwoByte);
1740 0 : masm.load32(newStartIndexAddress, base);
1741 2 : if (latin1)
1742 : masm.addPtr(temp2, base);
1743 : else
1744 2 : masm.computeEffectiveAddress(BaseIndex(temp2, base, TimesTwo), base);
1745 :
1746 2 : CopyStringChars(masm, string, base, temp1, temp2, latin1 ? 1 : 2, latin1 ? 1 : 2);
1747 :
1748 : // Null-terminate.
1749 0 : if (latin1)
1750 2 : masm.store8(Imm32(0), Address(string, 0));
1751 : else
1752 2 : masm.store16(Imm32(0), Address(string, 0));
1753 :
1754 0 : masm.pop(base);
1755 4 : masm.pop(string);
1756 : }
1757 :
1758 0 : masm.jump(&done);
1759 2 : masm.bind(¬Inline);
1760 :
1761 : {
1762 : // Make a dependent string.
1763 2 : int32_t flags = (latin1 ? JSString::LATIN1_CHARS_BIT : 0) | JSString::DEPENDENT_FLAGS;
1764 :
1765 4 : masm.newGCString(string, temp2, &fallbacks_[FallbackKind::NotInlineString], stringsCanBeInNursery);
1766 : // Warning: string may be tenured (if the fallback case is hit), so
1767 : // stores into it must be post barriered.
1768 0 : masm.bind(&joins_[FallbackKind::NotInlineString]);
1769 0 : masm.store32(Imm32(flags), Address(string, JSString::offsetOfFlags()));
1770 4 : masm.store32(temp1, Address(string, JSString::offsetOfLength()));
1771 :
1772 : masm.loadNonInlineStringChars(base, temp1,
1773 0 : latin1 ? CharEncoding::Latin1 : CharEncoding::TwoByte);
1774 0 : masm.load32(startIndexAddress, temp2);
1775 2 : if (latin1)
1776 : masm.addPtr(temp2, temp1);
1777 : else
1778 0 : masm.computeEffectiveAddress(BaseIndex(temp1, temp2, TimesTwo), temp1);
1779 0 : masm.storeNonInlineStringChars(temp1, string);
1780 0 : masm.storeDependentStringBase(base, string);
1781 4 : masm.movePtr(base, temp1);
1782 :
1783 : // Follow any base pointer if the input is itself a dependent string.
1784 : // Watch for undepended strings, which have a base pointer but don't
1785 : // actually share their characters with it.
1786 0 : Label noBase;
1787 0 : masm.load32(Address(base, JSString::offsetOfFlags()), temp2);
1788 0 : masm.and32(Imm32(JSString::TYPE_FLAGS_MASK), temp2);
1789 0 : masm.branch32(Assembler::NotEqual, temp2, Imm32(JSString::DEPENDENT_FLAGS), &noBase);
1790 0 : masm.loadDependentStringBase(base, temp1);
1791 0 : masm.storeDependentStringBase(temp1, string);
1792 2 : masm.bind(&noBase);
1793 :
1794 : // Post-barrier the base store, whether it was the direct or indirect
1795 : // base (both will end up in temp1 here).
1796 0 : masm.branchPtrInNurseryChunk(Assembler::Equal, string, temp2, &done);
1797 2 : masm.branchPtrInNurseryChunk(Assembler::NotEqual, temp1, temp2, &done);
1798 :
1799 0 : LiveRegisterSet regsToSave(RegisterSet::Volatile());
1800 0 : regsToSave.takeUnchecked(temp1);
1801 0 : regsToSave.takeUnchecked(temp2);
1802 2 : regsToSave.addUnchecked(string);
1803 :
1804 2 : masm.PushRegsInMask(regsToSave);
1805 :
1806 4 : masm.mov(ImmPtr(runtime), temp1);
1807 :
1808 0 : masm.setupUnalignedABICall(temp2);
1809 0 : masm.passABIArg(temp1);
1810 0 : masm.passABIArg(string);
1811 2 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, PostWriteBarrier));
1812 :
1813 2 : masm.PopRegsInMask(regsToSave);
1814 : }
1815 :
1816 0 : masm.bind(&done);
1817 2 : }
1818 :
1819 : static void*
1820 0 : AllocateString(JSContext* cx)
1821 : {
1822 0 : AutoUnsafeCallWithABI unsafe;
1823 0 : return js::Allocate<JSString, NoGC>(cx, js::gc::TenuredHeap);
1824 : }
1825 :
1826 : static void*
1827 0 : AllocateFatInlineString(JSContext* cx)
1828 : {
1829 0 : AutoUnsafeCallWithABI unsafe;
1830 0 : return js::Allocate<JSFatInlineString, NoGC>(cx, js::gc::TenuredHeap);
1831 : }
1832 :
1833 : void
1834 2 : CreateDependentString::generateFallback(MacroAssembler& masm, LiveRegisterSet regsToSave)
1835 : {
1836 0 : regsToSave.take(string_);
1837 0 : regsToSave.take(temp_);
1838 0 : for (FallbackKind kind : mozilla::MakeEnumeratedRange(FallbackKind::Count)) {
1839 12 : masm.bind(&fallbacks_[kind]);
1840 :
1841 6 : masm.PushRegsInMask(regsToSave);
1842 :
1843 0 : masm.setupUnalignedABICall(string_);
1844 0 : masm.loadJSContext(string_);
1845 0 : masm.passABIArg(string_);
1846 10 : masm.callWithABI(kind == FallbackKind::FatInlineString
1847 : ? JS_FUNC_TO_DATA_PTR(void*, AllocateFatInlineString)
1848 0 : : JS_FUNC_TO_DATA_PTR(void*, AllocateString));
1849 6 : masm.storeCallPointerResult(string_);
1850 :
1851 6 : masm.PopRegsInMask(regsToSave);
1852 :
1853 12 : masm.branchPtr(Assembler::Equal, string_, ImmWord(0), failure_);
1854 :
1855 18 : masm.jump(&joins_[kind]);
1856 : }
1857 2 : }
1858 :
1859 : static void*
1860 0 : CreateMatchResultFallbackFunc(JSContext* cx, gc::AllocKind kind, size_t nDynamicSlots)
1861 : {
1862 0 : AutoUnsafeCallWithABI unsafe;
1863 : return js::Allocate<JSObject, NoGC>(cx, kind, nDynamicSlots, gc::DefaultHeap,
1864 0 : &ArrayObject::class_);
1865 : }
1866 :
1867 : static void
1868 1 : CreateMatchResultFallback(MacroAssembler& masm, LiveRegisterSet regsToSave,
1869 : Register object, Register temp2, Register temp5,
1870 : ArrayObject* templateObj, Label* fail)
1871 : {
1872 1 : MOZ_ASSERT(templateObj->group()->clasp() == &ArrayObject::class_);
1873 :
1874 0 : regsToSave.take(object);
1875 0 : regsToSave.take(temp2);
1876 0 : regsToSave.take(temp5);
1877 1 : masm.PushRegsInMask(regsToSave);
1878 :
1879 1 : masm.setupUnalignedABICall(object);
1880 :
1881 0 : masm.loadJSContext(object);
1882 0 : masm.passABIArg(object);
1883 0 : masm.move32(Imm32(int32_t(templateObj->asTenured().getAllocKind())), temp2);
1884 0 : masm.passABIArg(temp2);
1885 0 : masm.move32(Imm32(int32_t(templateObj->as<NativeObject>().numDynamicSlots())), temp5);
1886 0 : masm.passABIArg(temp5);
1887 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, CreateMatchResultFallbackFunc));
1888 1 : masm.storeCallPointerResult(object);
1889 :
1890 1 : masm.PopRegsInMask(regsToSave);
1891 :
1892 2 : masm.branchPtr(Assembler::Equal, object, ImmWord(0), fail);
1893 :
1894 0 : TemplateObject templateObject(templateObj);
1895 0 : masm.initGCThing(object, temp2, templateObject, true);
1896 1 : }
1897 :
1898 : JitCode*
1899 1 : JitRealm::generateRegExpMatcherStub(JSContext* cx)
1900 : {
1901 0 : Register regexp = RegExpMatcherRegExpReg;
1902 0 : Register input = RegExpMatcherStringReg;
1903 0 : Register lastIndex = RegExpMatcherLastIndexReg;
1904 1 : ValueOperand result = JSReturnOperand;
1905 :
1906 : // We are free to clobber all registers, as LRegExpMatcher is a call instruction.
1907 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
1908 0 : regs.take(input);
1909 0 : regs.take(regexp);
1910 1 : regs.take(lastIndex);
1911 :
1912 : // temp5 is used in single byte instructions when creating dependent
1913 : // strings, and has restrictions on which register it can be on some
1914 : // platforms.
1915 1 : Register temp5;
1916 : {
1917 : AllocatableGeneralRegisterSet oregs = regs;
1918 : do {
1919 0 : temp5 = oregs.takeAny();
1920 0 : } while (!MacroAssembler::canUseInSingleByteInstruction(temp5));
1921 1 : regs.take(temp5);
1922 : }
1923 :
1924 0 : Register temp1 = regs.takeAny();
1925 0 : Register temp2 = regs.takeAny();
1926 1 : Register temp3 = regs.takeAny();
1927 :
1928 0 : Register maybeTemp4 = InvalidReg;
1929 1 : if (!regs.empty()) {
1930 : // There are not enough registers on x86.
1931 1 : maybeTemp4 = regs.takeAny();
1932 : }
1933 :
1934 0 : ArrayObject* templateObject = cx->realm()->regExps.getOrCreateMatchResultTemplateObject(cx);
1935 1 : if (!templateObject)
1936 : return nullptr;
1937 :
1938 : // The template object should have enough space for the maximum number of
1939 : // pairs this stub can handle.
1940 2 : MOZ_ASSERT(ObjectElements::VALUES_PER_HEADER + RegExpObject::MaxPairCount ==
1941 : gc::GetGCKindSlots(templateObject->asTenured().getAllocKind()));
1942 :
1943 1 : StackMacroAssembler masm(cx);
1944 :
1945 : // The InputOutputData is placed above the return address on the stack.
1946 1 : size_t inputOutputDataStartOffset = sizeof(void*);
1947 :
1948 0 : Label notFound, oolEntry;
1949 1 : if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex,
1950 : temp1, temp2, temp5, inputOutputDataStartOffset,
1951 1 : RegExpShared::Normal, stringsCanBeInNursery, ¬Found, &oolEntry))
1952 : {
1953 : return nullptr;
1954 : }
1955 :
1956 : // Construct the result.
1957 0 : Register object = temp1;
1958 0 : Label matchResultFallback, matchResultJoin;
1959 0 : TemplateObject templateObj(templateObject);
1960 0 : masm.createGCObject(object, temp2, templateObj, gc::DefaultHeap, &matchResultFallback);
1961 1 : masm.bind(&matchResultJoin);
1962 :
1963 : // Initialize slots of result object.
1964 0 : masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
1965 0 : masm.storeValue(templateObject->getSlot(0), Address(temp2, 0));
1966 1 : masm.storeValue(templateObject->getSlot(1), Address(temp2, sizeof(Value)));
1967 :
1968 1 : size_t elementsOffset = NativeObject::offsetOfFixedElements();
1969 :
1970 : #ifdef DEBUG
1971 : // Assert the initial value of initializedLength and length to make sure
1972 : // restoration on failure case works.
1973 : {
1974 0 : Label initLengthOK, lengthOK;
1975 0 : masm.branch32(Assembler::Equal,
1976 0 : Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength()),
1977 0 : Imm32(templateObject->getDenseInitializedLength()),
1978 0 : &initLengthOK);
1979 0 : masm.assumeUnreachable("Initial value of the match object's initializedLength does not match to restoration.");
1980 1 : masm.bind(&initLengthOK);
1981 :
1982 0 : masm.branch32(Assembler::Equal,
1983 0 : Address(object, elementsOffset + ObjectElements::offsetOfLength()),
1984 0 : Imm32(templateObject->length()),
1985 0 : &lengthOK);
1986 0 : masm.assumeUnreachable("Initial value of The match object's length does not match to restoration.");
1987 1 : masm.bind(&lengthOK);
1988 : }
1989 : #endif
1990 :
1991 0 : Register matchIndex = temp2;
1992 2 : masm.move32(Imm32(0), matchIndex);
1993 :
1994 0 : size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
1995 0 : Address pairsVectorAddress(masm.getStackPointer(), pairsVectorStartOffset);
1996 2 : Address pairCountAddress = RegExpPairCountAddress(masm, inputOutputDataStartOffset);
1997 :
1998 2 : BaseIndex stringAddress(object, matchIndex, TimesEight, elementsOffset);
1999 :
2000 : JS_STATIC_ASSERT(sizeof(MatchPair) == 8);
2001 : BaseIndex stringIndexAddress(masm.getStackPointer(), matchIndex, TimesEight,
2002 2 : pairsVectorStartOffset + offsetof(MatchPair, start));
2003 : BaseIndex stringLimitAddress(masm.getStackPointer(), matchIndex, TimesEight,
2004 2 : pairsVectorStartOffset + offsetof(MatchPair, limit));
2005 :
2006 : // Loop to construct the match strings. There are two different loops,
2007 : // depending on whether the input is latin1.
2008 8 : CreateDependentString depStr[2];
2009 :
2010 : // depStr may refer to failureRestore during generateFallback below,
2011 : // so this variable must live outside of the block.
2012 2 : Label failureRestore;
2013 : {
2014 0 : Label isLatin1, done;
2015 1 : masm.branchLatin1String(input, &isLatin1);
2016 :
2017 0 : Label* failure = &oolEntry;
2018 1 : Register temp4 = (maybeTemp4 == InvalidReg) ? lastIndex : maybeTemp4;
2019 :
2020 1 : if (maybeTemp4 == InvalidReg) {
2021 0 : failure = &failureRestore;
2022 :
2023 : // Save lastIndex value to temporary space.
2024 0 : masm.store32(lastIndex, Address(object, elementsOffset + ObjectElements::offsetOfLength()));
2025 : }
2026 :
2027 0 : for (int isLatin = 0; isLatin <= 1; isLatin++) {
2028 0 : if (isLatin)
2029 1 : masm.bind(&isLatin1);
2030 :
2031 0 : Label matchLoop;
2032 2 : masm.bind(&matchLoop);
2033 :
2034 0 : Label isUndefined, storeDone;
2035 2 : masm.branch32(Assembler::LessThan, stringIndexAddress, Imm32(0), &isUndefined);
2036 :
2037 6 : depStr[isLatin].generate(masm, cx->names(),
2038 : CompileRuntime::get(cx->runtime()),
2039 : isLatin, temp3, input, temp4, temp5,
2040 : stringIndexAddress, stringLimitAddress,
2041 0 : stringsCanBeInNursery,
2042 2 : failure);
2043 :
2044 2 : masm.storeValue(JSVAL_TYPE_STRING, temp3, stringAddress);
2045 : // Storing into nursery-allocated results object's elements; no post barrier.
2046 0 : masm.jump(&storeDone);
2047 2 : masm.bind(&isUndefined);
2048 :
2049 0 : masm.storeValue(UndefinedValue(), stringAddress);
2050 2 : masm.bind(&storeDone);
2051 :
2052 0 : masm.add32(Imm32(1), matchIndex);
2053 0 : masm.branch32(Assembler::LessThanOrEqual, pairCountAddress, matchIndex, &done);
2054 2 : masm.jump(&matchLoop);
2055 : }
2056 :
2057 1 : if (maybeTemp4 == InvalidReg) {
2058 : // Restore lastIndex value from temporary space, both for success
2059 : // and failure cases.
2060 :
2061 0 : masm.load32(Address(object, elementsOffset + ObjectElements::offsetOfLength()), lastIndex);
2062 0 : masm.jump(&done);
2063 :
2064 0 : masm.bind(&failureRestore);
2065 0 : masm.load32(Address(object, elementsOffset + ObjectElements::offsetOfLength()), lastIndex);
2066 :
2067 : // Restore the match object for failure case.
2068 0 : masm.store32(Imm32(templateObject->getDenseInitializedLength()),
2069 0 : Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength()));
2070 0 : masm.store32(Imm32(templateObject->length()),
2071 0 : Address(object, elementsOffset + ObjectElements::offsetOfLength()));
2072 : masm.jump(&oolEntry);
2073 : }
2074 :
2075 1 : masm.bind(&done);
2076 : }
2077 :
2078 : // Fill in the rest of the output object.
2079 0 : masm.store32(matchIndex, Address(object, elementsOffset + ObjectElements::offsetOfInitializedLength()));
2080 2 : masm.store32(matchIndex, Address(object, elementsOffset + ObjectElements::offsetOfLength()));
2081 :
2082 2 : masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
2083 :
2084 0 : MOZ_ASSERT(templateObject->numFixedSlots() == 0);
2085 0 : MOZ_ASSERT(templateObject->lookupPure(cx->names().index)->slot() == 0);
2086 2 : MOZ_ASSERT(templateObject->lookupPure(cx->names().input)->slot() == 1);
2087 :
2088 0 : masm.load32(pairsVectorAddress, temp3);
2089 0 : masm.storeValue(JSVAL_TYPE_INT32, temp3, Address(temp2, 0));
2090 0 : Address inputSlotAddress(temp2, sizeof(Value));
2091 1 : masm.storeValue(JSVAL_TYPE_STRING, input, inputSlotAddress);
2092 : // No post barrier needed (inputSlotAddress is within nursery object.)
2093 :
2094 : // All done!
2095 0 : masm.tagValue(JSVAL_TYPE_OBJECT, object, result);
2096 1 : masm.ret();
2097 :
2098 0 : masm.bind(¬Found);
2099 0 : masm.moveValue(NullValue(), result);
2100 1 : masm.ret();
2101 :
2102 : // Fallback paths for CreateDependentString and createGCObject.
2103 : // Need to save all registers in use when they were called.
2104 0 : LiveRegisterSet regsToSave(RegisterSet::Volatile());
2105 0 : regsToSave.addUnchecked(regexp);
2106 0 : regsToSave.addUnchecked(input);
2107 0 : regsToSave.addUnchecked(lastIndex);
2108 0 : regsToSave.addUnchecked(temp1);
2109 0 : regsToSave.addUnchecked(temp2);
2110 0 : regsToSave.addUnchecked(temp3);
2111 1 : if (maybeTemp4 != InvalidReg)
2112 : regsToSave.addUnchecked(maybeTemp4);
2113 1 : regsToSave.addUnchecked(temp5);
2114 :
2115 0 : for (int isLatin = 0; isLatin <= 1; isLatin++)
2116 2 : depStr[isLatin].generateFallback(masm, regsToSave);
2117 :
2118 0 : masm.bind(&matchResultFallback);
2119 0 : CreateMatchResultFallback(masm, regsToSave, object, temp2, temp5, templateObject, &oolEntry);
2120 1 : masm.jump(&matchResultJoin);
2121 :
2122 : // Use an undefined value to signal to the caller that the OOL stub needs to be called.
2123 0 : masm.bind(&oolEntry);
2124 0 : masm.moveValue(UndefinedValue(), result);
2125 1 : masm.ret();
2126 :
2127 0 : Linker linker(masm);
2128 0 : AutoFlushICache afc("RegExpMatcherStub");
2129 0 : JitCode* code = linker.newCode(cx, CodeKind::Other);
2130 1 : if (!code)
2131 : return nullptr;
2132 :
2133 : #ifdef JS_ION_PERF
2134 : writePerfSpewerJitCodeProfile(code, "RegExpMatcherStub");
2135 : #endif
2136 : #ifdef MOZ_VTUNE
2137 : vtune::MarkStub(code, "RegExpMatcherStub");
2138 : #endif
2139 :
2140 1 : return code;
2141 : }
2142 :
2143 : class OutOfLineRegExpMatcher : public OutOfLineCodeBase<CodeGenerator>
2144 : {
2145 : LRegExpMatcher* lir_;
2146 :
2147 : public:
2148 : explicit OutOfLineRegExpMatcher(LRegExpMatcher* lir)
2149 2 : : lir_(lir)
2150 : { }
2151 :
2152 0 : void accept(CodeGenerator* codegen) override {
2153 0 : codegen->visitOutOfLineRegExpMatcher(this);
2154 1 : }
2155 :
2156 : LRegExpMatcher* lir() const {
2157 : return lir_;
2158 : }
2159 : };
2160 :
2161 : typedef bool (*RegExpMatcherRawFn)(JSContext* cx, HandleObject regexp, HandleString input,
2162 : int32_t lastIndex,
2163 : MatchPairs* pairs, MutableHandleValue output);
2164 0 : static const VMFunction RegExpMatcherRawInfo =
2165 3 : FunctionInfo<RegExpMatcherRawFn>(RegExpMatcherRaw, "RegExpMatcherRaw");
2166 :
2167 : void
2168 1 : CodeGenerator::visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher* ool)
2169 : {
2170 0 : LRegExpMatcher* lir = ool->lir();
2171 0 : Register lastIndex = ToRegister(lir->lastIndex());
2172 0 : Register input = ToRegister(lir->string());
2173 2 : Register regexp = ToRegister(lir->regexp());
2174 :
2175 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
2176 0 : regs.take(lastIndex);
2177 0 : regs.take(input);
2178 0 : regs.take(regexp);
2179 1 : Register temp = regs.takeAny();
2180 :
2181 0 : masm.computeEffectiveAddress(Address(masm.getStackPointer(),
2182 1 : sizeof(irregexp::InputOutputData)), temp);
2183 :
2184 0 : pushArg(temp);
2185 0 : pushArg(lastIndex);
2186 0 : pushArg(input);
2187 2 : pushArg(regexp);
2188 :
2189 : // We are not using oolCallVM because we are in a Call, and that live
2190 : // registers are already saved by the the register allocator.
2191 1 : callVM(RegExpMatcherRawInfo, lir);
2192 :
2193 0 : masm.jump(ool->rejoin());
2194 1 : }
2195 :
2196 : void
2197 1 : CodeGenerator::visitRegExpMatcher(LRegExpMatcher* lir)
2198 : {
2199 0 : MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg);
2200 0 : MOZ_ASSERT(ToRegister(lir->string()) == RegExpMatcherStringReg);
2201 0 : MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg);
2202 2 : MOZ_ASSERT(ToOutValue(lir) == JSReturnOperand);
2203 :
2204 : #if defined(JS_NUNBOX32)
2205 : MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg_Type);
2206 : MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg_Data);
2207 : MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg_Type);
2208 : MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg_Data);
2209 : MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg_Type);
2210 : MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg_Data);
2211 : #elif defined(JS_PUNBOX64)
2212 : MOZ_ASSERT(RegExpMatcherRegExpReg != JSReturnReg);
2213 : MOZ_ASSERT(RegExpMatcherStringReg != JSReturnReg);
2214 : MOZ_ASSERT(RegExpMatcherLastIndexReg != JSReturnReg);
2215 : #endif
2216 :
2217 2 : masm.reserveStack(RegExpReservedStack);
2218 :
2219 0 : OutOfLineRegExpMatcher* ool = new(alloc()) OutOfLineRegExpMatcher(lir);
2220 2 : addOutOfLineCode(ool, lir->mir());
2221 :
2222 0 : const JitRealm* jitRealm = gen->realm->jitRealm();
2223 0 : JitCode* regExpMatcherStub = jitRealm->regExpMatcherStubNoBarrier(&realmStubsToReadBarrier_);
2224 0 : masm.call(regExpMatcherStub);
2225 0 : masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, ool->entry());
2226 2 : masm.bind(ool->rejoin());
2227 :
2228 0 : masm.freeStack(RegExpReservedStack);
2229 1 : }
2230 :
2231 : static const int32_t RegExpSearcherResultNotFound = -1;
2232 : static const int32_t RegExpSearcherResultFailed = -2;
2233 :
2234 : JitCode*
2235 1 : JitRealm::generateRegExpSearcherStub(JSContext* cx)
2236 : {
2237 0 : Register regexp = RegExpTesterRegExpReg;
2238 0 : Register input = RegExpTesterStringReg;
2239 0 : Register lastIndex = RegExpTesterLastIndexReg;
2240 1 : Register result = ReturnReg;
2241 :
2242 : // We are free to clobber all registers, as LRegExpSearcher is a call instruction.
2243 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
2244 0 : regs.take(input);
2245 0 : regs.take(regexp);
2246 1 : regs.take(lastIndex);
2247 :
2248 0 : Register temp1 = regs.takeAny();
2249 0 : Register temp2 = regs.takeAny();
2250 1 : Register temp3 = regs.takeAny();
2251 :
2252 2 : StackMacroAssembler masm(cx);
2253 :
2254 : // The InputOutputData is placed above the return address on the stack.
2255 1 : size_t inputOutputDataStartOffset = sizeof(void*);
2256 :
2257 0 : Label notFound, oolEntry;
2258 1 : if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex,
2259 : temp1, temp2, temp3, inputOutputDataStartOffset,
2260 1 : RegExpShared::Normal, stringsCanBeInNursery,
2261 : ¬Found, &oolEntry))
2262 : {
2263 : return nullptr;
2264 : }
2265 :
2266 1 : size_t pairsVectorStartOffset = RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
2267 : Address stringIndexAddress(masm.getStackPointer(),
2268 2 : pairsVectorStartOffset + offsetof(MatchPair, start));
2269 : Address stringLimitAddress(masm.getStackPointer(),
2270 2 : pairsVectorStartOffset + offsetof(MatchPair, limit));
2271 :
2272 0 : masm.load32(stringIndexAddress, result);
2273 0 : masm.load32(stringLimitAddress, input);
2274 0 : masm.lshiftPtr(Imm32(15), input);
2275 0 : masm.or32(input, result);
2276 1 : masm.ret();
2277 :
2278 0 : masm.bind(¬Found);
2279 0 : masm.move32(Imm32(RegExpSearcherResultNotFound), result);
2280 1 : masm.ret();
2281 :
2282 0 : masm.bind(&oolEntry);
2283 0 : masm.move32(Imm32(RegExpSearcherResultFailed), result);
2284 1 : masm.ret();
2285 :
2286 0 : Linker linker(masm);
2287 0 : AutoFlushICache afc("RegExpSearcherStub");
2288 0 : JitCode* code = linker.newCode(cx, CodeKind::Other);
2289 1 : if (!code)
2290 : return nullptr;
2291 :
2292 : #ifdef JS_ION_PERF
2293 : writePerfSpewerJitCodeProfile(code, "RegExpSearcherStub");
2294 : #endif
2295 : #ifdef MOZ_VTUNE
2296 : vtune::MarkStub(code, "RegExpSearcherStub");
2297 : #endif
2298 :
2299 1 : return code;
2300 : }
2301 :
2302 : class OutOfLineRegExpSearcher : public OutOfLineCodeBase<CodeGenerator>
2303 : {
2304 : LRegExpSearcher* lir_;
2305 :
2306 : public:
2307 : explicit OutOfLineRegExpSearcher(LRegExpSearcher* lir)
2308 2 : : lir_(lir)
2309 : { }
2310 :
2311 0 : void accept(CodeGenerator* codegen) override {
2312 0 : codegen->visitOutOfLineRegExpSearcher(this);
2313 1 : }
2314 :
2315 : LRegExpSearcher* lir() const {
2316 : return lir_;
2317 : }
2318 : };
2319 :
2320 : typedef bool (*RegExpSearcherRawFn)(JSContext* cx, HandleObject regexp, HandleString input,
2321 : int32_t lastIndex,
2322 : MatchPairs* pairs, int32_t* result);
2323 0 : static const VMFunction RegExpSearcherRawInfo =
2324 3 : FunctionInfo<RegExpSearcherRawFn>(RegExpSearcherRaw, "RegExpSearcherRaw");
2325 :
2326 : void
2327 1 : CodeGenerator::visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher* ool)
2328 : {
2329 0 : LRegExpSearcher* lir = ool->lir();
2330 0 : Register lastIndex = ToRegister(lir->lastIndex());
2331 0 : Register input = ToRegister(lir->string());
2332 2 : Register regexp = ToRegister(lir->regexp());
2333 :
2334 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
2335 0 : regs.take(lastIndex);
2336 0 : regs.take(input);
2337 0 : regs.take(regexp);
2338 1 : Register temp = regs.takeAny();
2339 :
2340 0 : masm.computeEffectiveAddress(Address(masm.getStackPointer(),
2341 1 : sizeof(irregexp::InputOutputData)), temp);
2342 :
2343 0 : pushArg(temp);
2344 0 : pushArg(lastIndex);
2345 0 : pushArg(input);
2346 2 : pushArg(regexp);
2347 :
2348 : // We are not using oolCallVM because we are in a Call, and that live
2349 : // registers are already saved by the the register allocator.
2350 1 : callVM(RegExpSearcherRawInfo, lir);
2351 :
2352 0 : masm.jump(ool->rejoin());
2353 1 : }
2354 :
2355 : void
2356 1 : CodeGenerator::visitRegExpSearcher(LRegExpSearcher* lir)
2357 : {
2358 0 : MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpTesterRegExpReg);
2359 0 : MOZ_ASSERT(ToRegister(lir->string()) == RegExpTesterStringReg);
2360 0 : MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpTesterLastIndexReg);
2361 2 : MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg);
2362 :
2363 : MOZ_ASSERT(RegExpTesterRegExpReg != ReturnReg);
2364 : MOZ_ASSERT(RegExpTesterStringReg != ReturnReg);
2365 : MOZ_ASSERT(RegExpTesterLastIndexReg != ReturnReg);
2366 :
2367 2 : masm.reserveStack(RegExpReservedStack);
2368 :
2369 0 : OutOfLineRegExpSearcher* ool = new(alloc()) OutOfLineRegExpSearcher(lir);
2370 2 : addOutOfLineCode(ool, lir->mir());
2371 :
2372 0 : const JitRealm* jitRealm = gen->realm->jitRealm();
2373 0 : JitCode* regExpSearcherStub = jitRealm->regExpSearcherStubNoBarrier(&realmStubsToReadBarrier_);
2374 0 : masm.call(regExpSearcherStub);
2375 0 : masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpSearcherResultFailed), ool->entry());
2376 2 : masm.bind(ool->rejoin());
2377 :
2378 0 : masm.freeStack(RegExpReservedStack);
2379 1 : }
2380 :
2381 : static const int32_t RegExpTesterResultNotFound = -1;
2382 : static const int32_t RegExpTesterResultFailed = -2;
2383 :
2384 : JitCode*
2385 1 : JitRealm::generateRegExpTesterStub(JSContext* cx)
2386 : {
2387 0 : Register regexp = RegExpTesterRegExpReg;
2388 0 : Register input = RegExpTesterStringReg;
2389 0 : Register lastIndex = RegExpTesterLastIndexReg;
2390 1 : Register result = ReturnReg;
2391 :
2392 2 : StackMacroAssembler masm(cx);
2393 :
2394 : #ifdef JS_USE_LINK_REGISTER
2395 : masm.pushReturnAddress();
2396 : #endif
2397 :
2398 : // We are free to clobber all registers, as LRegExpTester is a call instruction.
2399 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
2400 0 : regs.take(input);
2401 0 : regs.take(regexp);
2402 1 : regs.take(lastIndex);
2403 :
2404 0 : Register temp1 = regs.takeAny();
2405 0 : Register temp2 = regs.takeAny();
2406 1 : Register temp3 = regs.takeAny();
2407 :
2408 1 : masm.reserveStack(sizeof(irregexp::InputOutputData));
2409 :
2410 0 : Label notFound, oolEntry;
2411 1 : if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex,
2412 : temp1, temp2, temp3, 0,
2413 1 : RegExpShared::MatchOnly, stringsCanBeInNursery,
2414 : ¬Found, &oolEntry))
2415 : {
2416 : return nullptr;
2417 : }
2418 :
2419 2 : Label done;
2420 :
2421 : // temp3 contains endIndex.
2422 0 : masm.move32(temp3, result);
2423 1 : masm.jump(&done);
2424 :
2425 0 : masm.bind(¬Found);
2426 0 : masm.move32(Imm32(RegExpTesterResultNotFound), result);
2427 1 : masm.jump(&done);
2428 :
2429 0 : masm.bind(&oolEntry);
2430 2 : masm.move32(Imm32(RegExpTesterResultFailed), result);
2431 :
2432 0 : masm.bind(&done);
2433 0 : masm.freeStack(sizeof(irregexp::InputOutputData));
2434 1 : masm.ret();
2435 :
2436 0 : Linker linker(masm);
2437 0 : AutoFlushICache afc("RegExpTesterStub");
2438 0 : JitCode* code = linker.newCode(cx, CodeKind::Other);
2439 1 : if (!code)
2440 : return nullptr;
2441 :
2442 : #ifdef JS_ION_PERF
2443 : writePerfSpewerJitCodeProfile(code, "RegExpTesterStub");
2444 : #endif
2445 : #ifdef MOZ_VTUNE
2446 : vtune::MarkStub(code, "RegExpTesterStub");
2447 : #endif
2448 :
2449 1 : return code;
2450 : }
2451 :
2452 : class OutOfLineRegExpTester : public OutOfLineCodeBase<CodeGenerator>
2453 : {
2454 : LRegExpTester* lir_;
2455 :
2456 : public:
2457 : explicit OutOfLineRegExpTester(LRegExpTester* lir)
2458 8 : : lir_(lir)
2459 : { }
2460 :
2461 0 : void accept(CodeGenerator* codegen) override {
2462 0 : codegen->visitOutOfLineRegExpTester(this);
2463 4 : }
2464 :
2465 : LRegExpTester* lir() const {
2466 : return lir_;
2467 : }
2468 : };
2469 :
2470 : typedef bool (*RegExpTesterRawFn)(JSContext* cx, HandleObject regexp, HandleString input,
2471 : int32_t lastIndex, int32_t* result);
2472 0 : static const VMFunction RegExpTesterRawInfo =
2473 3 : FunctionInfo<RegExpTesterRawFn>(RegExpTesterRaw, "RegExpTesterRaw");
2474 :
2475 : void
2476 4 : CodeGenerator::visitOutOfLineRegExpTester(OutOfLineRegExpTester* ool)
2477 : {
2478 0 : LRegExpTester* lir = ool->lir();
2479 0 : Register lastIndex = ToRegister(lir->lastIndex());
2480 0 : Register input = ToRegister(lir->string());
2481 8 : Register regexp = ToRegister(lir->regexp());
2482 :
2483 0 : pushArg(lastIndex);
2484 0 : pushArg(input);
2485 8 : pushArg(regexp);
2486 :
2487 : // We are not using oolCallVM because we are in a Call, and that live
2488 : // registers are already saved by the the register allocator.
2489 4 : callVM(RegExpTesterRawInfo, lir);
2490 :
2491 0 : masm.jump(ool->rejoin());
2492 4 : }
2493 :
2494 : void
2495 4 : CodeGenerator::visitRegExpTester(LRegExpTester* lir)
2496 : {
2497 0 : MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpTesterRegExpReg);
2498 0 : MOZ_ASSERT(ToRegister(lir->string()) == RegExpTesterStringReg);
2499 0 : MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpTesterLastIndexReg);
2500 8 : MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg);
2501 :
2502 : MOZ_ASSERT(RegExpTesterRegExpReg != ReturnReg);
2503 : MOZ_ASSERT(RegExpTesterStringReg != ReturnReg);
2504 : MOZ_ASSERT(RegExpTesterLastIndexReg != ReturnReg);
2505 :
2506 0 : OutOfLineRegExpTester* ool = new(alloc()) OutOfLineRegExpTester(lir);
2507 8 : addOutOfLineCode(ool, lir->mir());
2508 :
2509 0 : const JitRealm* jitRealm = gen->realm->jitRealm();
2510 0 : JitCode* regExpTesterStub = jitRealm->regExpTesterStubNoBarrier(&realmStubsToReadBarrier_);
2511 4 : masm.call(regExpTesterStub);
2512 :
2513 0 : masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpTesterResultFailed), ool->entry());
2514 0 : masm.bind(ool->rejoin());
2515 4 : }
2516 :
2517 : class OutOfLineRegExpPrototypeOptimizable : public OutOfLineCodeBase<CodeGenerator>
2518 : {
2519 : LRegExpPrototypeOptimizable* ins_;
2520 :
2521 : public:
2522 : explicit OutOfLineRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable* ins)
2523 0 : : ins_(ins)
2524 : { }
2525 :
2526 0 : void accept(CodeGenerator* codegen) override {
2527 0 : codegen->visitOutOfLineRegExpPrototypeOptimizable(this);
2528 0 : }
2529 : LRegExpPrototypeOptimizable* ins() const {
2530 : return ins_;
2531 : }
2532 : };
2533 :
2534 : void
2535 0 : CodeGenerator::visitRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable* ins)
2536 : {
2537 0 : Register object = ToRegister(ins->object());
2538 0 : Register output = ToRegister(ins->output());
2539 0 : Register temp = ToRegister(ins->temp());
2540 :
2541 0 : OutOfLineRegExpPrototypeOptimizable* ool = new(alloc()) OutOfLineRegExpPrototypeOptimizable(ins);
2542 0 : addOutOfLineCode(ool, ins->mir());
2543 :
2544 0 : masm.loadJSContext(temp);
2545 0 : masm.loadPtr(Address(temp, JSContext::offsetOfRealm()), temp);
2546 : size_t offset = Realm::offsetOfRegExps() +
2547 0 : RegExpRealm::offsetOfOptimizableRegExpPrototypeShape();
2548 0 : masm.loadPtr(Address(temp, offset), temp);
2549 :
2550 0 : masm.branchTestObjShapeUnsafe(Assembler::NotEqual, object, temp, ool->entry());
2551 0 : masm.move32(Imm32(0x1), output);
2552 :
2553 0 : masm.bind(ool->rejoin());
2554 0 : }
2555 :
2556 : void
2557 0 : CodeGenerator::visitOutOfLineRegExpPrototypeOptimizable(OutOfLineRegExpPrototypeOptimizable* ool)
2558 : {
2559 0 : LRegExpPrototypeOptimizable* ins = ool->ins();
2560 0 : Register object = ToRegister(ins->object());
2561 0 : Register output = ToRegister(ins->output());
2562 :
2563 0 : saveVolatile(output);
2564 :
2565 0 : masm.setupUnalignedABICall(output);
2566 0 : masm.loadJSContext(output);
2567 0 : masm.passABIArg(output);
2568 0 : masm.passABIArg(object);
2569 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, RegExpPrototypeOptimizableRaw));
2570 0 : masm.storeCallBoolResult(output);
2571 :
2572 0 : restoreVolatile(output);
2573 :
2574 0 : masm.jump(ool->rejoin());
2575 0 : }
2576 :
2577 : class OutOfLineRegExpInstanceOptimizable : public OutOfLineCodeBase<CodeGenerator>
2578 : {
2579 : LRegExpInstanceOptimizable* ins_;
2580 :
2581 : public:
2582 : explicit OutOfLineRegExpInstanceOptimizable(LRegExpInstanceOptimizable* ins)
2583 0 : : ins_(ins)
2584 : { }
2585 :
2586 0 : void accept(CodeGenerator* codegen) override {
2587 0 : codegen->visitOutOfLineRegExpInstanceOptimizable(this);
2588 0 : }
2589 : LRegExpInstanceOptimizable* ins() const {
2590 : return ins_;
2591 : }
2592 : };
2593 :
2594 : void
2595 0 : CodeGenerator::visitRegExpInstanceOptimizable(LRegExpInstanceOptimizable* ins)
2596 : {
2597 0 : Register object = ToRegister(ins->object());
2598 0 : Register output = ToRegister(ins->output());
2599 0 : Register temp = ToRegister(ins->temp());
2600 :
2601 0 : OutOfLineRegExpInstanceOptimizable* ool = new(alloc()) OutOfLineRegExpInstanceOptimizable(ins);
2602 0 : addOutOfLineCode(ool, ins->mir());
2603 :
2604 0 : masm.loadJSContext(temp);
2605 0 : masm.loadPtr(Address(temp, JSContext::offsetOfRealm()), temp);
2606 : size_t offset = Realm::offsetOfRegExps() +
2607 0 : RegExpRealm::offsetOfOptimizableRegExpInstanceShape();
2608 0 : masm.loadPtr(Address(temp, offset), temp);
2609 :
2610 0 : masm.branchTestObjShapeUnsafe(Assembler::NotEqual, object, temp, ool->entry());
2611 0 : masm.move32(Imm32(0x1), output);
2612 :
2613 0 : masm.bind(ool->rejoin());
2614 0 : }
2615 :
2616 : void
2617 0 : CodeGenerator::visitOutOfLineRegExpInstanceOptimizable(OutOfLineRegExpInstanceOptimizable* ool)
2618 : {
2619 0 : LRegExpInstanceOptimizable* ins = ool->ins();
2620 0 : Register object = ToRegister(ins->object());
2621 0 : Register proto = ToRegister(ins->proto());
2622 0 : Register output = ToRegister(ins->output());
2623 :
2624 0 : saveVolatile(output);
2625 :
2626 0 : masm.setupUnalignedABICall(output);
2627 0 : masm.loadJSContext(output);
2628 0 : masm.passABIArg(output);
2629 0 : masm.passABIArg(object);
2630 0 : masm.passABIArg(proto);
2631 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, RegExpInstanceOptimizableRaw));
2632 0 : masm.storeCallBoolResult(output);
2633 :
2634 0 : restoreVolatile(output);
2635 :
2636 0 : masm.jump(ool->rejoin());
2637 0 : }
2638 :
2639 : static void
2640 0 : FindFirstDollarIndex(MacroAssembler& masm, Register len, Register chars,
2641 : Register temp, Register output, bool isLatin1)
2642 : {
2643 0 : masm.move32(Imm32(0), output);
2644 :
2645 0 : Label start, done;
2646 0 : masm.bind(&start);
2647 0 : if (isLatin1)
2648 0 : masm.load8ZeroExtend(BaseIndex(chars, output, TimesOne), temp);
2649 : else
2650 0 : masm.load16ZeroExtend(BaseIndex(chars, output, TimesTwo), temp);
2651 :
2652 0 : masm.branch32(Assembler::Equal, temp, Imm32('$'), &done);
2653 :
2654 0 : masm.add32(Imm32(1), output);
2655 0 : masm.branch32(Assembler::NotEqual, output, len, &start);
2656 :
2657 0 : masm.move32(Imm32(-1), output);
2658 :
2659 0 : masm.bind(&done);
2660 0 : }
2661 :
2662 : typedef bool (*GetFirstDollarIndexRawFn)(JSContext*, JSString*, int32_t*);
2663 0 : static const VMFunction GetFirstDollarIndexRawInfo =
2664 3 : FunctionInfo<GetFirstDollarIndexRawFn>(GetFirstDollarIndexRaw, "GetFirstDollarIndexRaw");
2665 :
2666 : void
2667 0 : CodeGenerator::visitGetFirstDollarIndex(LGetFirstDollarIndex* ins)
2668 : {
2669 0 : Register str = ToRegister(ins->str());
2670 0 : Register output = ToRegister(ins->output());
2671 0 : Register temp0 = ToRegister(ins->temp0());
2672 0 : Register temp1 = ToRegister(ins->temp1());
2673 0 : Register len = ToRegister(ins->temp2());
2674 :
2675 0 : OutOfLineCode* ool = oolCallVM(GetFirstDollarIndexRawInfo, ins, ArgList(str),
2676 0 : StoreRegisterTo(output));
2677 :
2678 0 : masm.branchIfRope(str, ool->entry());
2679 0 : masm.loadStringLength(str, len);
2680 :
2681 0 : Label isLatin1, done;
2682 0 : masm.branchLatin1String(str, &isLatin1);
2683 : {
2684 0 : masm.loadStringChars(str, temp0, CharEncoding::TwoByte);
2685 0 : FindFirstDollarIndex(masm, len, temp0, temp1, output, /* isLatin1 = */ false);
2686 0 : masm.jump(&done);
2687 : }
2688 0 : masm.bind(&isLatin1);
2689 : {
2690 0 : masm.loadStringChars(str, temp0, CharEncoding::Latin1);
2691 0 : FindFirstDollarIndex(masm, len, temp0, temp1, output, /* isLatin1 = */ true);
2692 : }
2693 0 : masm.bind(&done);
2694 0 : masm.bind(ool->rejoin());
2695 0 : }
2696 :
2697 : typedef JSString* (*StringReplaceFn)(JSContext*, HandleString, HandleString, HandleString);
2698 0 : static const VMFunction StringFlatReplaceInfo =
2699 0 : FunctionInfo<StringReplaceFn>(js::str_flat_replace_string, "str_flat_replace_string");
2700 0 : static const VMFunction StringReplaceInfo =
2701 3 : FunctionInfo<StringReplaceFn>(StringReplace, "StringReplace");
2702 :
2703 : void
2704 0 : CodeGenerator::visitStringReplace(LStringReplace* lir)
2705 : {
2706 0 : if (lir->replacement()->isConstant())
2707 0 : pushArg(ImmGCPtr(lir->replacement()->toConstant()->toString()));
2708 : else
2709 0 : pushArg(ToRegister(lir->replacement()));
2710 :
2711 0 : if (lir->pattern()->isConstant())
2712 0 : pushArg(ImmGCPtr(lir->pattern()->toConstant()->toString()));
2713 : else
2714 0 : pushArg(ToRegister(lir->pattern()));
2715 :
2716 0 : if (lir->string()->isConstant())
2717 0 : pushArg(ImmGCPtr(lir->string()->toConstant()->toString()));
2718 : else
2719 0 : pushArg(ToRegister(lir->string()));
2720 :
2721 0 : if (lir->mir()->isFlatReplacement())
2722 0 : callVM(StringFlatReplaceInfo, lir);
2723 : else
2724 0 : callVM(StringReplaceInfo, lir);
2725 0 : }
2726 :
2727 : void
2728 15 : CodeGenerator::emitSharedStub(ICStub::Kind kind, LInstruction* lir)
2729 : {
2730 0 : JSScript* script = lir->mirRaw()->block()->info().script();
2731 15 : jsbytecode* pc = lir->mirRaw()->toInstruction()->resumePoint()->pc();
2732 :
2733 : #ifdef JS_USE_LINK_REGISTER
2734 : // Some architectures don't push the return address on the stack but
2735 : // use the link register. In that case the stack isn't aligned. Push
2736 : // to make sure we are aligned.
2737 : masm.Push(Imm32(0));
2738 : #endif
2739 :
2740 : // Create descriptor signifying end of Ion frame.
2741 0 : uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS,
2742 0 : JitStubFrameLayout::Size());
2743 30 : masm.Push(Imm32(descriptor));
2744 :
2745 : // Call into the stubcode.
2746 0 : CodeOffset patchOffset;
2747 0 : IonICEntry entry(script->pcToOffset(pc), ICEntry::Kind_Op, script);
2748 0 : EmitCallIC(&patchOffset, masm);
2749 45 : entry.setReturnOffset(CodeOffset(masm.currentOffset()));
2750 :
2751 0 : SharedStub sharedStub(kind, entry, patchOffset);
2752 30 : masm.propagateOOM(sharedStubs_.append(sharedStub));
2753 :
2754 : // Fix up upon return.
2755 30 : uint32_t callOffset = masm.currentOffset();
2756 : #ifdef JS_USE_LINK_REGISTER
2757 : masm.freeStack(sizeof(intptr_t) * 2);
2758 : #else
2759 15 : masm.freeStack(sizeof(intptr_t));
2760 : #endif
2761 0 : markSafepointAt(callOffset, lir);
2762 15 : }
2763 :
2764 : void
2765 6 : CodeGenerator::visitBinarySharedStub(LBinarySharedStub* lir)
2766 : {
2767 0 : JSOp jsop = JSOp(*lir->mirRaw()->toInstruction()->resumePoint()->pc());
2768 6 : switch (jsop) {
2769 : case JSOP_ADD:
2770 : case JSOP_SUB:
2771 : case JSOP_MUL:
2772 : case JSOP_DIV:
2773 : case JSOP_MOD:
2774 : case JSOP_POW:
2775 0 : emitSharedStub(ICStub::Kind::BinaryArith_Fallback, lir);
2776 1 : break;
2777 : case JSOP_LT:
2778 : case JSOP_LE:
2779 : case JSOP_GT:
2780 : case JSOP_GE:
2781 : case JSOP_EQ:
2782 : case JSOP_NE:
2783 : case JSOP_STRICTEQ:
2784 : case JSOP_STRICTNE:
2785 0 : emitSharedStub(ICStub::Kind::Compare_Fallback, lir);
2786 5 : break;
2787 : default:
2788 0 : MOZ_CRASH("Unsupported jsop in shared stubs.");
2789 : }
2790 6 : }
2791 :
2792 : void
2793 0 : CodeGenerator::visitUnaryCache(LUnaryCache* lir)
2794 : {
2795 0 : LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
2796 0 : TypedOrValueRegister input = TypedOrValueRegister(ToValue(lir, LUnaryCache::Input));
2797 0 : ValueOperand output = ToOutValue(lir);
2798 :
2799 0 : IonUnaryArithIC ic(liveRegs, input, output);
2800 0 : addIC(lir, allocateIC(ic));
2801 0 : }
2802 :
2803 : void
2804 9 : CodeGenerator::visitNullarySharedStub(LNullarySharedStub* lir)
2805 : {
2806 0 : jsbytecode* pc = lir->mir()->resumePoint()->pc();
2807 0 : JSOp jsop = JSOp(*pc);
2808 9 : switch (jsop) {
2809 : case JSOP_NEWARRAY: {
2810 0 : uint32_t length = GET_UINT32(pc);
2811 8 : MOZ_ASSERT(length <= INT32_MAX,
2812 : "the bytecode emitter must fail to compile code that would "
2813 : "produce JSOP_NEWARRAY with a length exceeding int32_t range");
2814 :
2815 : // Pass length in R0.
2816 0 : masm.move32(Imm32(AssertedCast<int32_t>(length)), R0.scratchReg());
2817 0 : emitSharedStub(ICStub::Kind::NewArray_Fallback, lir);
2818 8 : break;
2819 : }
2820 : case JSOP_NEWOBJECT:
2821 0 : emitSharedStub(ICStub::Kind::NewObject_Fallback, lir);
2822 1 : break;
2823 : case JSOP_NEWINIT: {
2824 0 : JSProtoKey key = JSProtoKey(GET_UINT8(pc));
2825 0 : if (key == JSProto_Array) {
2826 0 : masm.move32(Imm32(0), R0.scratchReg());
2827 0 : emitSharedStub(ICStub::Kind::NewArray_Fallback, lir);
2828 : } else {
2829 0 : emitSharedStub(ICStub::Kind::NewObject_Fallback, lir);
2830 : }
2831 : break;
2832 : }
2833 : default:
2834 0 : MOZ_CRASH("Unsupported jsop in shared stubs.");
2835 : }
2836 9 : }
2837 :
2838 : typedef JSFunction* (*MakeDefaultConstructorFn)(JSContext*, HandleScript,
2839 : jsbytecode*, HandleObject);
2840 0 : static const VMFunction MakeDefaultConstructorInfo =
2841 0 : FunctionInfo<MakeDefaultConstructorFn>(js::MakeDefaultConstructor,
2842 1 : "MakeDefaultConstructor");
2843 :
2844 : void
2845 0 : CodeGenerator::visitClassConstructor(LClassConstructor* lir)
2846 : {
2847 0 : pushArg(ImmPtr(nullptr));
2848 0 : pushArg(ImmPtr(lir->mir()->pc()));
2849 0 : pushArg(ImmGCPtr(current->mir()->info().script()));
2850 0 : callVM(MakeDefaultConstructorInfo, lir);
2851 0 : }
2852 :
2853 : typedef JSObject* (*LambdaFn)(JSContext*, HandleFunction, HandleObject);
2854 2 : static const VMFunction LambdaInfo = FunctionInfo<LambdaFn>(js::Lambda, "Lambda");
2855 :
2856 : void
2857 0 : CodeGenerator::visitLambdaForSingleton(LLambdaForSingleton* lir)
2858 : {
2859 0 : pushArg(ToRegister(lir->environmentChain()));
2860 0 : pushArg(ImmGCPtr(lir->mir()->info().funUnsafe()));
2861 0 : callVM(LambdaInfo, lir);
2862 0 : }
2863 :
2864 : void
2865 4 : CodeGenerator::visitLambda(LLambda* lir)
2866 : {
2867 0 : Register envChain = ToRegister(lir->environmentChain());
2868 0 : Register output = ToRegister(lir->output());
2869 0 : Register tempReg = ToRegister(lir->temp());
2870 12 : const LambdaFunctionInfo& info = lir->mir()->info();
2871 :
2872 0 : OutOfLineCode* ool = oolCallVM(LambdaInfo, lir, ArgList(ImmGCPtr(info.funUnsafe()), envChain),
2873 12 : StoreRegisterTo(output));
2874 :
2875 4 : MOZ_ASSERT(!info.singletonType);
2876 :
2877 0 : TemplateObject templateObject(info.funUnsafe());
2878 4 : masm.createGCObject(output, tempReg, templateObject, gc::DefaultHeap, ool->entry());
2879 :
2880 4 : emitLambdaInit(output, envChain, info);
2881 :
2882 4 : if (info.flags & JSFunction::EXTENDED) {
2883 : static_assert(FunctionExtended::NUM_EXTENDED_SLOTS == 2, "All slots must be initialized");
2884 0 : masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(0)));
2885 2 : masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(1)));
2886 : }
2887 :
2888 0 : masm.bind(ool->rejoin());
2889 4 : }
2890 :
2891 : class OutOfLineLambdaArrow : public OutOfLineCodeBase<CodeGenerator>
2892 : {
2893 : public:
2894 : LLambdaArrow* lir;
2895 : Label entryNoPop_;
2896 :
2897 : explicit OutOfLineLambdaArrow(LLambdaArrow* lir)
2898 0 : : lir(lir)
2899 : { }
2900 :
2901 0 : void accept(CodeGenerator* codegen) override {
2902 0 : codegen->visitOutOfLineLambdaArrow(this);
2903 0 : }
2904 :
2905 : Label* entryNoPop() {
2906 0 : return &entryNoPop_;
2907 : }
2908 : };
2909 :
2910 : typedef JSObject* (*LambdaArrowFn)(JSContext*, HandleFunction, HandleObject, HandleValue);
2911 0 : static const VMFunction LambdaArrowInfo =
2912 3 : FunctionInfo<LambdaArrowFn>(js::LambdaArrow, "LambdaArrow");
2913 :
2914 : void
2915 0 : CodeGenerator::visitOutOfLineLambdaArrow(OutOfLineLambdaArrow* ool)
2916 : {
2917 0 : Register envChain = ToRegister(ool->lir->environmentChain());
2918 0 : ValueOperand newTarget = ToValue(ool->lir, LLambdaArrow::NewTargetValue);
2919 0 : Register output = ToRegister(ool->lir->output());
2920 0 : const LambdaFunctionInfo& info = ool->lir->mir()->info();
2921 :
2922 : // When we get here, we may need to restore part of the newTarget,
2923 : // which has been conscripted into service as a temp register.
2924 0 : masm.pop(newTarget.scratchReg());
2925 :
2926 0 : masm.bind(ool->entryNoPop());
2927 :
2928 0 : saveLive(ool->lir);
2929 :
2930 0 : pushArg(newTarget);
2931 0 : pushArg(envChain);
2932 0 : pushArg(ImmGCPtr(info.funUnsafe()));
2933 :
2934 0 : callVM(LambdaArrowInfo, ool->lir);
2935 0 : StoreRegisterTo(output).generate(this);
2936 :
2937 0 : restoreLiveIgnore(ool->lir, StoreRegisterTo(output).clobbered());
2938 :
2939 0 : masm.jump(ool->rejoin());
2940 0 : }
2941 :
2942 : void
2943 0 : CodeGenerator::visitLambdaArrow(LLambdaArrow* lir)
2944 : {
2945 0 : Register envChain = ToRegister(lir->environmentChain());
2946 0 : ValueOperand newTarget = ToValue(lir, LLambdaArrow::NewTargetValue);
2947 0 : Register output = ToRegister(lir->output());
2948 0 : const LambdaFunctionInfo& info = lir->mir()->info();
2949 :
2950 0 : OutOfLineLambdaArrow* ool = new (alloc()) OutOfLineLambdaArrow(lir);
2951 0 : addOutOfLineCode(ool, lir->mir());
2952 :
2953 0 : MOZ_ASSERT(!info.useSingletonForClone);
2954 :
2955 0 : if (info.singletonType) {
2956 : // If the function has a singleton type, this instruction will only be
2957 : // executed once so we don't bother inlining it.
2958 0 : masm.jump(ool->entryNoPop());
2959 0 : masm.bind(ool->rejoin());
2960 0 : return;
2961 : }
2962 :
2963 : // There's not enough registers on x86 with the profiler enabled to request
2964 : // a temp. Instead, spill part of one of the values, being prepared to
2965 : // restore it if necessary on the out of line path.
2966 0 : Register tempReg = newTarget.scratchReg();
2967 0 : masm.push(newTarget.scratchReg());
2968 :
2969 0 : TemplateObject templateObject(info.funUnsafe());
2970 0 : masm.createGCObject(output, tempReg, templateObject, gc::DefaultHeap, ool->entry());
2971 :
2972 0 : masm.pop(newTarget.scratchReg());
2973 :
2974 0 : emitLambdaInit(output, envChain, info);
2975 :
2976 : // Initialize extended slots. Lexical |this| is stored in the first one.
2977 0 : MOZ_ASSERT(info.flags & JSFunction::EXTENDED);
2978 : static_assert(FunctionExtended::NUM_EXTENDED_SLOTS == 2, "All slots must be initialized");
2979 : static_assert(FunctionExtended::ARROW_NEWTARGET_SLOT == 0,
2980 : "|new.target| must be stored in first slot");
2981 0 : masm.storeValue(newTarget, Address(output, FunctionExtended::offsetOfExtendedSlot(0)));
2982 0 : masm.storeValue(UndefinedValue(), Address(output, FunctionExtended::offsetOfExtendedSlot(1)));
2983 :
2984 0 : masm.bind(ool->rejoin());
2985 : }
2986 :
2987 : void
2988 4 : CodeGenerator::emitLambdaInit(Register output, Register envChain,
2989 : const LambdaFunctionInfo& info)
2990 : {
2991 : // Initialize nargs and flags. We do this with a single uint32 to avoid
2992 : // 16-bit writes.
2993 : union {
2994 : struct S {
2995 : uint16_t nargs;
2996 : uint16_t flags;
2997 : } s;
2998 : uint32_t word;
2999 : } u;
3000 0 : u.s.nargs = info.nargs;
3001 4 : u.s.flags = info.flags;
3002 :
3003 : static_assert(JSFunction::offsetOfFlags() == JSFunction::offsetOfNargs() + 2,
3004 : "the code below needs to be adapted");
3005 0 : masm.store32(Imm32(u.word), Address(output, JSFunction::offsetOfNargs()));
3006 0 : masm.storePtr(ImmGCPtr(info.scriptOrLazyScript),
3007 0 : Address(output, JSFunction::offsetOfScriptOrLazyScript()));
3008 8 : masm.storePtr(envChain, Address(output, JSFunction::offsetOfEnvironment()));
3009 : // No post barrier needed because output is guaranteed to be allocated in
3010 : // the nursery.
3011 0 : masm.storePtr(ImmGCPtr(info.funUnsafe()->displayAtom()),
3012 0 : Address(output, JSFunction::offsetOfAtom()));
3013 4 : }
3014 :
3015 : typedef bool (*SetFunNameFn)(JSContext*, HandleFunction, HandleValue, FunctionPrefixKind);
3016 0 : static const VMFunction SetFunNameInfo =
3017 3 : FunctionInfo<SetFunNameFn>(js::SetFunctionNameIfNoOwnName, "SetFunName");
3018 :
3019 : void
3020 0 : CodeGenerator::visitSetFunName(LSetFunName* lir)
3021 : {
3022 0 : pushArg(Imm32(lir->mir()->prefixKind()));
3023 0 : pushArg(ToValue(lir, LSetFunName::NameValue));
3024 0 : pushArg(ToRegister(lir->fun()));
3025 :
3026 0 : callVM(SetFunNameInfo, lir);
3027 0 : }
3028 :
3029 : void
3030 1276 : CodeGenerator::visitOsiPoint(LOsiPoint* lir)
3031 : {
3032 : // Note: markOsiPoint ensures enough space exists between the last
3033 : // LOsiPoint and this one to patch adjacent call instructions.
3034 :
3035 2552 : MOZ_ASSERT(masm.framePushed() == frameSize());
3036 :
3037 1276 : uint32_t osiCallPointOffset = markOsiPoint(lir);
3038 :
3039 0 : LSafepoint* safepoint = lir->associatedSafepoint();
3040 0 : MOZ_ASSERT(!safepoint->osiCallPointOffset());
3041 1276 : safepoint->setOsiCallPointOffset(osiCallPointOffset);
3042 :
3043 : #ifdef DEBUG
3044 : // There should be no movegroups or other instructions between
3045 : // an instruction and its OsiPoint. This is necessary because
3046 : // we use the OsiPoint's snapshot from within VM calls.
3047 0 : for (LInstructionReverseIterator iter(current->rbegin(lir)); iter != current->rend(); iter++) {
3048 2552 : if (*iter == lir)
3049 : continue;
3050 0 : MOZ_ASSERT(!iter->isMoveGroup());
3051 1276 : MOZ_ASSERT(iter->safepoint() == safepoint);
3052 : break;
3053 : }
3054 : #endif
3055 :
3056 : #ifdef CHECK_OSIPOINT_REGISTERS
3057 0 : if (shouldVerifyOsiPointRegs(safepoint))
3058 0 : verifyOsiPointRegs(safepoint);
3059 : #endif
3060 1276 : }
3061 :
3062 : void
3063 0 : CodeGenerator::visitPhi(LPhi* lir)
3064 : {
3065 0 : MOZ_CRASH("Unexpected LPhi in CodeGenerator");
3066 : }
3067 :
3068 : void
3069 0 : CodeGenerator::visitGoto(LGoto* lir)
3070 : {
3071 1 : jumpToBlock(lir->target());
3072 0 : }
3073 :
3074 : typedef bool (*InterruptCheckFn)(JSContext*);
3075 0 : static const VMFunction InterruptCheckInfo =
3076 3 : FunctionInfo<InterruptCheckFn>(InterruptCheck, "InterruptCheck");
3077 :
3078 : void
3079 0 : CodeGenerator::visitTableSwitch(LTableSwitch* ins)
3080 : {
3081 0 : MTableSwitch* mir = ins->mir();
3082 0 : Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
3083 : const LAllocation* temp;
3084 :
3085 0 : if (mir->getOperand(0)->type() != MIRType::Int32) {
3086 0 : temp = ins->tempInt()->output();
3087 :
3088 : // The input is a double, so try and convert it to an integer.
3089 : // If it does not fit in an integer, take the default case.
3090 0 : masm.convertDoubleToInt32(ToFloatRegister(ins->index()), ToRegister(temp), defaultcase, false);
3091 : } else {
3092 0 : temp = ins->index();
3093 : }
3094 :
3095 0 : emitTableSwitchDispatch(mir, ToRegister(temp), ToRegisterOrInvalid(ins->tempPointer()));
3096 0 : }
3097 :
3098 : void
3099 0 : CodeGenerator::visitTableSwitchV(LTableSwitchV* ins)
3100 : {
3101 0 : MTableSwitch* mir = ins->mir();
3102 0 : Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
3103 :
3104 0 : Register index = ToRegister(ins->tempInt());
3105 0 : ValueOperand value = ToValue(ins, LTableSwitchV::InputValue);
3106 0 : Register tag = masm.extractTag(value, index);
3107 0 : masm.branchTestNumber(Assembler::NotEqual, tag, defaultcase);
3108 :
3109 0 : Label unboxInt, isInt;
3110 0 : masm.branchTestInt32(Assembler::Equal, tag, &unboxInt);
3111 : {
3112 0 : FloatRegister floatIndex = ToFloatRegister(ins->tempFloat());
3113 0 : masm.unboxDouble(value, floatIndex);
3114 0 : masm.convertDoubleToInt32(floatIndex, index, defaultcase, false);
3115 0 : masm.jump(&isInt);
3116 : }
3117 :
3118 0 : masm.bind(&unboxInt);
3119 0 : masm.unboxInt32(value, index);
3120 :
3121 0 : masm.bind(&isInt);
3122 :
3123 0 : emitTableSwitchDispatch(mir, index, ToRegisterOrInvalid(ins->tempPointer()));
3124 0 : }
3125 :
3126 : typedef JSObject* (*DeepCloneObjectLiteralFn)(JSContext*, HandleObject, NewObjectKind);
3127 0 : static const VMFunction DeepCloneObjectLiteralInfo =
3128 3 : FunctionInfo<DeepCloneObjectLiteralFn>(DeepCloneObjectLiteral, "DeepCloneObjectLiteral");
3129 :
3130 : void
3131 0 : CodeGenerator::visitCloneLiteral(LCloneLiteral* lir)
3132 : {
3133 0 : pushArg(ImmWord(TenuredObject));
3134 0 : pushArg(ToRegister(lir->getObjectLiteral()));
3135 0 : callVM(DeepCloneObjectLiteralInfo, lir);
3136 0 : }
3137 :
3138 : void
3139 0 : CodeGenerator::visitParameter(LParameter* lir)
3140 : {
3141 0 : }
3142 :
3143 : void
3144 16 : CodeGenerator::visitCallee(LCallee* lir)
3145 : {
3146 0 : Register callee = ToRegister(lir->output());
3147 48 : Address ptr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken());
3148 :
3149 0 : masm.loadFunctionFromCalleeToken(ptr, callee);
3150 16 : }
3151 :
3152 : void
3153 0 : CodeGenerator::visitIsConstructing(LIsConstructing* lir)
3154 : {
3155 0 : Register output = ToRegister(lir->output());
3156 0 : Address calleeToken(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken());
3157 0 : masm.loadPtr(calleeToken, output);
3158 :
3159 : // We must be inside a function.
3160 0 : MOZ_ASSERT(current->mir()->info().script()->functionNonDelazifying());
3161 :
3162 : // The low bit indicates whether this call is constructing, just clear the
3163 : // other bits.
3164 : static_assert(CalleeToken_Function == 0x0, "CalleeTokenTag value should match");
3165 : static_assert(CalleeToken_FunctionConstructing == 0x1, "CalleeTokenTag value should match");
3166 0 : masm.andPtr(Imm32(0x1), output);
3167 0 : }
3168 :
3169 : void
3170 0 : CodeGenerator::visitStart(LStart* lir)
3171 : {
3172 0 : }
3173 :
3174 : void
3175 83 : CodeGenerator::visitReturn(LReturn* lir)
3176 : {
3177 : #if defined(JS_NUNBOX32)
3178 : DebugOnly<LAllocation*> type = lir->getOperand(TYPE_INDEX);
3179 : DebugOnly<LAllocation*> payload = lir->getOperand(PAYLOAD_INDEX);
3180 : MOZ_ASSERT(ToRegister(type) == JSReturnReg_Type);
3181 : MOZ_ASSERT(ToRegister(payload) == JSReturnReg_Data);
3182 : #elif defined(JS_PUNBOX64)
3183 0 : DebugOnly<LAllocation*> result = lir->getOperand(0);
3184 166 : MOZ_ASSERT(ToRegister(result) == JSReturnReg);
3185 : #endif
3186 : // Don't emit a jump to the return label if this is the last block.
3187 0 : if (current->mir() != *gen->graph().poBegin())
3188 0 : masm.jump(&returnLabel_);
3189 83 : }
3190 :
3191 : void
3192 1 : CodeGenerator::visitOsrEntry(LOsrEntry* lir)
3193 : {
3194 2 : Register temp = ToRegister(lir->temp());
3195 :
3196 : // Remember the OSR entry offset into the code buffer.
3197 0 : masm.flushBuffer();
3198 2 : setOsrEntryOffset(masm.size());
3199 :
3200 : #ifdef JS_TRACE_LOGGING
3201 0 : emitTracelogStopEvent(TraceLogger_Baseline);
3202 2 : emitTracelogStartEvent(TraceLogger_IonMonkey);
3203 : #endif
3204 :
3205 : // If profiling, save the current frame pointer to a per-thread global field.
3206 0 : if (isProfilerInstrumentationEnabled())
3207 0 : masm.profilerEnterFrame(masm.getStackPointer(), temp);
3208 :
3209 : // Allocate the full frame for this function
3210 : // Note we have a new entry here. So we reset MacroAssembler::framePushed()
3211 : // to 0, before reserving the stack.
3212 0 : MOZ_ASSERT(masm.framePushed() == frameSize());
3213 2 : masm.setFramePushed(0);
3214 :
3215 : // Ensure that the Ion frames is properly aligned.
3216 1 : masm.assertStackAlignment(JitStackAlignment, 0);
3217 :
3218 0 : masm.reserveStack(frameSize());
3219 1 : }
3220 :
3221 : void
3222 1 : CodeGenerator::visitOsrEnvironmentChain(LOsrEnvironmentChain* lir)
3223 : {
3224 0 : const LAllocation* frame = lir->getOperand(0);
3225 2 : const LDefinition* object = lir->getDef(0);
3226 :
3227 1 : const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfEnvironmentChain();
3228 :
3229 0 : masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));
3230 1 : }
3231 :
3232 : void
3233 0 : CodeGenerator::visitOsrArgumentsObject(LOsrArgumentsObject* lir)
3234 : {
3235 0 : const LAllocation* frame = lir->getOperand(0);
3236 0 : const LDefinition* object = lir->getDef(0);
3237 :
3238 0 : const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfArgsObj();
3239 :
3240 0 : masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));
3241 0 : }
3242 :
3243 : void
3244 6 : CodeGenerator::visitOsrValue(LOsrValue* value)
3245 : {
3246 0 : const LAllocation* frame = value->getOperand(0);
3247 6 : const ValueOperand out = ToOutValue(value);
3248 :
3249 12 : const ptrdiff_t frameOffset = value->mir()->frameOffset();
3250 :
3251 0 : masm.loadValue(Address(ToRegister(frame), frameOffset), out);
3252 6 : }
3253 :
3254 : void
3255 1 : CodeGenerator::visitOsrReturnValue(LOsrReturnValue* lir)
3256 : {
3257 0 : const LAllocation* frame = lir->getOperand(0);
3258 1 : const ValueOperand out = ToOutValue(lir);
3259 :
3260 0 : Address flags = Address(ToRegister(frame), BaselineFrame::reverseOffsetOfFlags());
3261 3 : Address retval = Address(ToRegister(frame), BaselineFrame::reverseOffsetOfReturnValue());
3262 :
3263 1 : masm.moveValue(UndefinedValue(), out);
3264 :
3265 0 : Label done;
3266 0 : masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL), &done);
3267 0 : masm.loadValue(retval, out);
3268 0 : masm.bind(&done);
3269 1 : }
3270 :
3271 : void
3272 473 : CodeGenerator::visitStackArgT(LStackArgT* lir)
3273 : {
3274 0 : const LAllocation* arg = lir->getArgument();
3275 0 : MIRType argType = lir->type();
3276 0 : uint32_t argslot = lir->argslot();
3277 473 : MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount());
3278 :
3279 0 : int32_t stack_offset = StackOffsetOfPassedArg(argslot);
3280 946 : Address dest(masm.getStackPointer(), stack_offset);
3281 :
3282 0 : if (arg->isFloatReg())
3283 0 : masm.storeDouble(ToFloatRegister(arg), dest);
3284 0 : else if (arg->isRegister())
3285 247 : masm.storeValue(ValueTypeFromMIRType(argType), ToRegister(arg), dest);
3286 : else
3287 0 : masm.storeValue(arg->toConstant()->toJSValue(), dest);
3288 473 : }
3289 :
3290 : void
3291 30 : CodeGenerator::visitStackArgV(LStackArgV* lir)
3292 : {
3293 0 : ValueOperand val = ToValue(lir, 0);
3294 0 : uint32_t argslot = lir->argslot();
3295 30 : MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount());
3296 :
3297 30 : int32_t stack_offset = StackOffsetOfPassedArg(argslot);
3298 :
3299 0 : masm.storeValue(val, Address(masm.getStackPointer(), stack_offset));
3300 30 : }
3301 :
3302 : void
3303 1979 : CodeGenerator::visitMoveGroup(LMoveGroup* group)
3304 : {
3305 0 : if (!group->numMoves())
3306 0 : return;
3307 :
3308 1979 : MoveResolver& resolver = masm.moveResolver();
3309 :
3310 0 : for (size_t i = 0; i < group->numMoves(); i++) {
3311 3121 : const LMove& move = group->getMove(i);
3312 :
3313 0 : LAllocation from = move.from();
3314 0 : LAllocation to = move.to();
3315 3121 : LDefinition::Type type = move.type();
3316 :
3317 : // No bogus moves.
3318 0 : MOZ_ASSERT(from != to);
3319 3121 : MOZ_ASSERT(!from.isConstant());
3320 : MoveOp::Type moveType;
3321 3121 : switch (type) {
3322 : case LDefinition::OBJECT:
3323 : case LDefinition::SLOTS:
3324 : #ifdef JS_NUNBOX32
3325 : case LDefinition::TYPE:
3326 : case LDefinition::PAYLOAD:
3327 : #else
3328 : case LDefinition::BOX:
3329 : #endif
3330 : case LDefinition::GENERAL: moveType = MoveOp::GENERAL; break;
3331 0 : case LDefinition::INT32: moveType = MoveOp::INT32; break;
3332 0 : case LDefinition::FLOAT32: moveType = MoveOp::FLOAT32; break;
3333 0 : case LDefinition::DOUBLE: moveType = MoveOp::DOUBLE; break;
3334 0 : case LDefinition::SIMD128INT: moveType = MoveOp::SIMD128INT; break;
3335 0 : case LDefinition::SIMD128FLOAT: moveType = MoveOp::SIMD128FLOAT; break;
3336 0 : default: MOZ_CRASH("Unexpected move type");
3337 : }
3338 :
3339 6242 : masm.propagateOOM(resolver.addMove(toMoveOperand(from), toMoveOperand(to), moveType));
3340 : }
3341 :
3342 0 : masm.propagateOOM(resolver.resolve());
3343 3958 : if (masm.oom())
3344 : return;
3345 :
3346 3958 : MoveEmitter emitter(masm);
3347 :
3348 : #ifdef JS_CODEGEN_X86
3349 : if (group->maybeScratchRegister().isGeneralReg())
3350 : emitter.setScratchRegister(group->maybeScratchRegister().toGeneralReg()->reg());
3351 : else
3352 : resolver.sortMemoryToMemoryMoves();
3353 : #endif
3354 :
3355 0 : emitter.emit(resolver);
3356 1979 : emitter.finish();
3357 : }
3358 :
3359 : void
3360 230 : CodeGenerator::visitInteger(LInteger* lir)
3361 : {
3362 0 : masm.move32(Imm32(lir->getValue()), ToRegister(lir->output()));
3363 230 : }
3364 :
3365 : void
3366 0 : CodeGenerator::visitInteger64(LInteger64* lir)
3367 : {
3368 0 : masm.move64(Imm64(lir->getValue()), ToOutRegister64(lir));
3369 0 : }
3370 :
3371 : void
3372 202 : CodeGenerator::visitPointer(LPointer* lir)
3373 : {
3374 0 : if (lir->kind() == LPointer::GC_THING)
3375 606 : masm.movePtr(ImmGCPtr(lir->gcptr()), ToRegister(lir->output()));
3376 : else
3377 0 : masm.movePtr(ImmPtr(lir->ptr()), ToRegister(lir->output()));
3378 202 : }
3379 :
3380 : void
3381 0 : CodeGenerator::visitKeepAliveObject(LKeepAliveObject* lir)
3382 : {
3383 : // No-op.
3384 0 : }
3385 :
3386 : void
3387 13 : CodeGenerator::visitSlots(LSlots* lir)
3388 : {
3389 0 : Address slots(ToRegister(lir->object()), NativeObject::offsetOfSlots());
3390 0 : masm.loadPtr(slots, ToRegister(lir->output()));
3391 13 : }
3392 :
3393 : void
3394 2 : CodeGenerator::visitLoadSlotT(LLoadSlotT* lir)
3395 : {
3396 0 : Register base = ToRegister(lir->slots());
3397 0 : int32_t offset = lir->mir()->slot() * sizeof(js::Value);
3398 4 : AnyRegister result = ToAnyRegister(lir->output());
3399 :
3400 0 : masm.loadUnboxedValue(Address(base, offset), lir->mir()->type(), result);
3401 2 : }
3402 :
3403 : void
3404 11 : CodeGenerator::visitLoadSlotV(LLoadSlotV* lir)
3405 : {
3406 0 : ValueOperand dest = ToOutValue(lir);
3407 0 : Register base = ToRegister(lir->input());
3408 22 : int32_t offset = lir->mir()->slot() * sizeof(js::Value);
3409 :
3410 0 : masm.loadValue(Address(base, offset), dest);
3411 11 : }
3412 :
3413 : void
3414 0 : CodeGenerator::visitStoreSlotT(LStoreSlotT* lir)
3415 : {
3416 0 : Register base = ToRegister(lir->slots());
3417 0 : int32_t offset = lir->mir()->slot() * sizeof(js::Value);
3418 0 : Address dest(base, offset);
3419 :
3420 0 : if (lir->mir()->needsBarrier())
3421 0 : emitPreBarrier(dest);
3422 :
3423 0 : MIRType valueType = lir->mir()->value()->type();
3424 :
3425 0 : if (valueType == MIRType::ObjectOrNull) {
3426 0 : masm.storeObjectOrNull(ToRegister(lir->value()), dest);
3427 : } else {
3428 0 : ConstantOrRegister value;
3429 0 : if (lir->value()->isConstant())
3430 0 : value = ConstantOrRegister(lir->value()->toConstant()->toJSValue());
3431 : else
3432 0 : value = TypedOrValueRegister(valueType, ToAnyRegister(lir->value()));
3433 0 : masm.storeUnboxedValue(value, valueType, dest, lir->mir()->slotType());
3434 : }
3435 0 : }
3436 :
3437 : void
3438 0 : CodeGenerator::visitStoreSlotV(LStoreSlotV* lir)
3439 : {
3440 0 : Register base = ToRegister(lir->slots());
3441 0 : int32_t offset = lir->mir()->slot() * sizeof(Value);
3442 :
3443 0 : const ValueOperand value = ToValue(lir, LStoreSlotV::Value);
3444 :
3445 0 : if (lir->mir()->needsBarrier())
3446 0 : emitPreBarrier(Address(base, offset));
3447 :
3448 0 : masm.storeValue(value, Address(base, offset));
3449 0 : }
3450 :
3451 : static void
3452 22 : GuardReceiver(MacroAssembler& masm, const ReceiverGuard& guard,
3453 : Register obj, Register expandoScratch, Register scratch, Label* miss,
3454 : bool checkNullExpando)
3455 : {
3456 0 : if (guard.group) {
3457 2 : masm.branchTestObjGroup(Assembler::NotEqual, obj, guard.group, scratch, obj, miss);
3458 :
3459 0 : Address expandoAddress(obj, UnboxedPlainObject::offsetOfExpando());
3460 0 : if (guard.shape) {
3461 0 : masm.loadPtr(expandoAddress, expandoScratch);
3462 0 : masm.branchPtr(Assembler::Equal, expandoScratch, ImmWord(0), miss);
3463 0 : masm.branchTestObjShape(Assembler::NotEqual, expandoScratch, guard.shape, scratch,
3464 0 : expandoScratch, miss);
3465 0 : } else if (checkNullExpando) {
3466 0 : masm.branchPtr(Assembler::NotEqual, expandoAddress, ImmWord(0), miss);
3467 : }
3468 : } else {
3469 20 : masm.branchTestObjShape(Assembler::NotEqual, obj, guard.shape, scratch, obj, miss);
3470 : }
3471 22 : }
3472 :
3473 : void
3474 6 : CodeGenerator::emitGetPropertyPolymorphic(LInstruction* ins, Register obj, Register expandoScratch,
3475 : Register scratch,
3476 : const TypedOrValueRegister& output)
3477 : {
3478 12 : MGetPropertyPolymorphic* mir = ins->mirRaw()->toGetPropertyPolymorphic();
3479 :
3480 12 : Label done;
3481 :
3482 0 : for (size_t i = 0; i < mir->numReceivers(); i++) {
3483 16 : ReceiverGuard receiver = mir->receiver(i);
3484 :
3485 0 : Label next;
3486 0 : masm.comment("GuardReceiver");
3487 0 : GuardReceiver(masm, receiver, obj, expandoScratch, scratch, &next,
3488 16 : /* checkNullExpando = */ false);
3489 :
3490 0 : if (receiver.shape) {
3491 14 : masm.comment("loadTypedOrValue");
3492 : // If this is an unboxed expando access, GuardReceiver loaded the
3493 : // expando object into expandoScratch.
3494 14 : Register target = receiver.group ? expandoScratch : obj;
3495 :
3496 0 : Shape* shape = mir->shape(i);
3497 28 : if (shape->slot() < shape->numFixedSlots()) {
3498 : // Fixed slot.
3499 0 : masm.loadTypedOrValue(Address(target, NativeObject::getFixedSlotOffset(shape->slot())),
3500 4 : output);
3501 : } else {
3502 : // Dynamic slot.
3503 0 : uint32_t offset = (shape->slot() - shape->numFixedSlots()) * sizeof(js::Value);
3504 0 : masm.loadPtr(Address(target, NativeObject::offsetOfSlots()), scratch);
3505 20 : masm.loadTypedOrValue(Address(scratch, offset), output);
3506 : }
3507 : } else {
3508 2 : masm.comment("loadUnboxedProperty");
3509 : const UnboxedLayout::Property* property =
3510 0 : receiver.group->unboxedLayoutDontCheckGeneration().lookup(mir->name());
3511 4 : Address propertyAddr(obj, UnboxedPlainObject::offsetOfData() + property->offset);
3512 :
3513 2 : masm.loadUnboxedProperty(propertyAddr, property->type, output);
3514 : }
3515 :
3516 0 : if (i == mir->numReceivers() - 1) {
3517 6 : bailoutFrom(&next, ins->snapshot());
3518 : } else {
3519 0 : masm.jump(&done);
3520 10 : masm.bind(&next);
3521 : }
3522 : }
3523 :
3524 0 : masm.bind(&done);
3525 6 : }
3526 :
3527 : void
3528 3 : CodeGenerator::visitGetPropertyPolymorphicV(LGetPropertyPolymorphicV* ins)
3529 : {
3530 0 : Register obj = ToRegister(ins->obj());
3531 0 : ValueOperand output = ToOutValue(ins);
3532 0 : Register temp = ToRegister(ins->temp());
3533 0 : emitGetPropertyPolymorphic(ins, obj, output.scratchReg(), temp, output);
3534 3 : }
3535 :
3536 : void
3537 3 : CodeGenerator::visitGetPropertyPolymorphicT(LGetPropertyPolymorphicT* ins)
3538 : {
3539 0 : Register obj = ToRegister(ins->obj());
3540 0 : TypedOrValueRegister output(ins->mir()->type(), ToAnyRegister(ins->output()));
3541 0 : Register temp1 = ToRegister(ins->temp1());
3542 0 : Register temp2 = (output.type() == MIRType::Double)
3543 0 : ? ToRegister(ins->temp2())
3544 0 : : output.typedReg().gpr();
3545 0 : emitGetPropertyPolymorphic(ins, obj, temp1, temp2, output);
3546 3 : }
3547 :
3548 : template <typename T>
3549 : static void
3550 0 : EmitUnboxedPreBarrier(MacroAssembler &masm, T address, JSValueType type)
3551 : {
3552 0 : if (type == JSVAL_TYPE_OBJECT)
3553 0 : masm.guardedCallPreBarrier(address, MIRType::Object);
3554 0 : else if (type == JSVAL_TYPE_STRING)
3555 0 : masm.guardedCallPreBarrier(address, MIRType::String);
3556 : else
3557 : MOZ_ASSERT(!UnboxedTypeNeedsPreBarrier(type));
3558 0 : }
3559 :
3560 : void
3561 0 : CodeGenerator::emitSetPropertyPolymorphic(LInstruction* ins, Register obj, Register expandoScratch,
3562 : Register scratch, const ConstantOrRegister& value)
3563 : {
3564 0 : MSetPropertyPolymorphic* mir = ins->mirRaw()->toSetPropertyPolymorphic();
3565 :
3566 0 : Label done;
3567 0 : for (size_t i = 0; i < mir->numReceivers(); i++) {
3568 0 : ReceiverGuard receiver = mir->receiver(i);
3569 :
3570 0 : Label next;
3571 0 : GuardReceiver(masm, receiver, obj, expandoScratch, scratch, &next,
3572 0 : /* checkNullExpando = */ false);
3573 :
3574 0 : if (receiver.shape) {
3575 : // If this is an unboxed expando access, GuardReceiver loaded the
3576 : // expando object into expandoScratch.
3577 0 : Register target = receiver.group ? expandoScratch : obj;
3578 :
3579 0 : Shape* shape = mir->shape(i);
3580 0 : if (shape->slot() < shape->numFixedSlots()) {
3581 : // Fixed slot.
3582 0 : Address addr(target, NativeObject::getFixedSlotOffset(shape->slot()));
3583 0 : if (mir->needsBarrier())
3584 0 : emitPreBarrier(addr);
3585 0 : masm.storeConstantOrRegister(value, addr);
3586 : } else {
3587 : // Dynamic slot.
3588 0 : masm.loadPtr(Address(target, NativeObject::offsetOfSlots()), scratch);
3589 0 : Address addr(scratch, (shape->slot() - shape->numFixedSlots()) * sizeof(js::Value));
3590 0 : if (mir->needsBarrier())
3591 0 : emitPreBarrier(addr);
3592 0 : masm.storeConstantOrRegister(value, addr);
3593 : }
3594 : } else {
3595 : const UnboxedLayout::Property* property =
3596 0 : receiver.group->unboxedLayoutDontCheckGeneration().lookup(mir->name());
3597 0 : Address propertyAddr(obj, UnboxedPlainObject::offsetOfData() + property->offset);
3598 :
3599 0 : EmitUnboxedPreBarrier(masm, propertyAddr, property->type);
3600 0 : masm.storeUnboxedProperty(propertyAddr, property->type, value, nullptr);
3601 : }
3602 :
3603 0 : if (i == mir->numReceivers() - 1) {
3604 0 : bailoutFrom(&next, ins->snapshot());
3605 : } else {
3606 0 : masm.jump(&done);
3607 0 : masm.bind(&next);
3608 : }
3609 : }
3610 :
3611 0 : masm.bind(&done);
3612 0 : }
3613 :
3614 : void
3615 0 : CodeGenerator::visitSetPropertyPolymorphicV(LSetPropertyPolymorphicV* ins)
3616 : {
3617 0 : Register obj = ToRegister(ins->obj());
3618 0 : Register temp1 = ToRegister(ins->temp1());
3619 0 : Register temp2 = ToRegister(ins->temp2());
3620 0 : ValueOperand value = ToValue(ins, LSetPropertyPolymorphicV::Value);
3621 0 : emitSetPropertyPolymorphic(ins, obj, temp1, temp2, TypedOrValueRegister(value));
3622 0 : }
3623 :
3624 : void
3625 0 : CodeGenerator::visitSetPropertyPolymorphicT(LSetPropertyPolymorphicT* ins)
3626 : {
3627 0 : Register obj = ToRegister(ins->obj());
3628 0 : Register temp1 = ToRegister(ins->temp1());
3629 0 : Register temp2 = ToRegister(ins->temp2());
3630 :
3631 0 : ConstantOrRegister value;
3632 0 : if (ins->mir()->value()->isConstant())
3633 0 : value = ConstantOrRegister(ins->mir()->value()->toConstant()->toJSValue());
3634 : else
3635 0 : value = TypedOrValueRegister(ins->mir()->value()->type(), ToAnyRegister(ins->value()));
3636 :
3637 0 : emitSetPropertyPolymorphic(ins, obj, temp1, temp2, value);
3638 0 : }
3639 :
3640 : void
3641 160 : CodeGenerator::visitElements(LElements* lir)
3642 : {
3643 0 : Address elements(ToRegister(lir->object()), NativeObject::offsetOfElements());
3644 0 : masm.loadPtr(elements, ToRegister(lir->output()));
3645 160 : }
3646 :
3647 : typedef bool (*ConvertElementsToDoublesFn)(JSContext*, uintptr_t);
3648 0 : static const VMFunction ConvertElementsToDoublesInfo =
3649 0 : FunctionInfo<ConvertElementsToDoublesFn>(ObjectElements::ConvertElementsToDoubles,
3650 1 : "ObjectElements::ConvertElementsToDoubles");
3651 :
3652 : void
3653 0 : CodeGenerator::visitConvertElementsToDoubles(LConvertElementsToDoubles* lir)
3654 : {
3655 0 : Register elements = ToRegister(lir->elements());
3656 :
3657 0 : OutOfLineCode* ool = oolCallVM(ConvertElementsToDoublesInfo, lir,
3658 0 : ArgList(elements), StoreNothing());
3659 :
3660 0 : Address convertedAddress(elements, ObjectElements::offsetOfFlags());
3661 0 : Imm32 bit(ObjectElements::CONVERT_DOUBLE_ELEMENTS);
3662 0 : masm.branchTest32(Assembler::Zero, convertedAddress, bit, ool->entry());
3663 0 : masm.bind(ool->rejoin());
3664 0 : }
3665 :
3666 : void
3667 0 : CodeGenerator::visitMaybeToDoubleElement(LMaybeToDoubleElement* lir)
3668 : {
3669 0 : Register elements = ToRegister(lir->elements());
3670 0 : Register value = ToRegister(lir->value());
3671 0 : ValueOperand out = ToOutValue(lir);
3672 :
3673 0 : FloatRegister temp = ToFloatRegister(lir->tempFloat());
3674 0 : Label convert, done;
3675 :
3676 : // If the CONVERT_DOUBLE_ELEMENTS flag is set, convert the int32
3677 : // value to double. Else, just box it.
3678 0 : masm.branchTest32(Assembler::NonZero,
3679 0 : Address(elements, ObjectElements::offsetOfFlags()),
3680 : Imm32(ObjectElements::CONVERT_DOUBLE_ELEMENTS),
3681 0 : &convert);
3682 :
3683 0 : masm.tagValue(JSVAL_TYPE_INT32, value, out);
3684 0 : masm.jump(&done);
3685 :
3686 0 : masm.bind(&convert);
3687 0 : masm.convertInt32ToDouble(value, temp);
3688 0 : masm.boxDouble(temp, out, temp);
3689 :
3690 0 : masm.bind(&done);
3691 0 : }
3692 :
3693 : typedef bool (*CopyElementsForWriteFn)(JSContext*, NativeObject*);
3694 0 : static const VMFunction CopyElementsForWriteInfo =
3695 0 : FunctionInfo<CopyElementsForWriteFn>(NativeObject::CopyElementsForWrite,
3696 1 : "NativeObject::CopyElementsForWrite");
3697 :
3698 : void
3699 7 : CodeGenerator::visitMaybeCopyElementsForWrite(LMaybeCopyElementsForWrite* lir)
3700 : {
3701 0 : Register object = ToRegister(lir->object());
3702 14 : Register temp = ToRegister(lir->temp());
3703 :
3704 0 : OutOfLineCode* ool = oolCallVM(CopyElementsForWriteInfo, lir,
3705 14 : ArgList(object), StoreNothing());
3706 :
3707 0 : if (lir->mir()->checkNative())
3708 7 : masm.branchIfNonNativeObj(object, temp, ool->rejoin());
3709 :
3710 0 : masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp);
3711 0 : masm.branchTest32(Assembler::NonZero,
3712 21 : Address(temp, ObjectElements::offsetOfFlags()),
3713 : Imm32(ObjectElements::COPY_ON_WRITE),
3714 0 : ool->entry());
3715 0 : masm.bind(ool->rejoin());
3716 7 : }
3717 :
3718 : void
3719 24 : CodeGenerator::visitFunctionEnvironment(LFunctionEnvironment* lir)
3720 : {
3721 0 : Address environment(ToRegister(lir->function()), JSFunction::offsetOfEnvironment());
3722 0 : masm.loadPtr(environment, ToRegister(lir->output()));
3723 24 : }
3724 :
3725 : void
3726 2 : CodeGenerator::visitHomeObject(LHomeObject* lir)
3727 : {
3728 8 : Address homeObject(ToRegister(lir->function()), FunctionExtended::offsetOfMethodHomeObjectSlot());
3729 : #ifdef DEBUG
3730 0 : Label isObject;
3731 0 : masm.branchTestObject(Assembler::Equal, homeObject, &isObject);
3732 0 : masm.assumeUnreachable("[[HomeObject]] must be Object");
3733 2 : masm.bind(&isObject);
3734 : #endif
3735 0 : masm.unboxObject(homeObject, ToRegister(lir->output()));
3736 2 : }
3737 :
3738 : typedef JSObject* (*HomeObjectSuperBaseFn)(JSContext*, HandleObject);
3739 0 : static const VMFunction HomeObjectSuperBaseInfo =
3740 3 : FunctionInfo<HomeObjectSuperBaseFn>(HomeObjectSuperBase, "HomeObjectSuperBase");
3741 :
3742 : void
3743 2 : CodeGenerator::visitHomeObjectSuperBase(LHomeObjectSuperBase* lir)
3744 : {
3745 0 : Register homeObject = ToRegister(lir->homeObject());
3746 4 : Register output = ToRegister(lir->output());
3747 :
3748 0 : OutOfLineCode* ool = oolCallVM(HomeObjectSuperBaseInfo, lir, ArgList(homeObject),
3749 6 : StoreRegisterTo(output));
3750 :
3751 0 : masm.loadObjProto(homeObject, output);
3752 0 : masm.branchPtr(Assembler::BelowOrEqual, output, ImmWord(1), ool->entry());
3753 0 : masm.bind(ool->rejoin());
3754 2 : }
3755 :
3756 : typedef LexicalEnvironmentObject* (*NewLexicalEnvironmentObjectFn)(JSContext*,
3757 : Handle<LexicalScope*>,
3758 : HandleObject, gc::InitialHeap);
3759 0 : static const VMFunction NewLexicalEnvironmentObjectInfo =
3760 0 : FunctionInfo<NewLexicalEnvironmentObjectFn>(LexicalEnvironmentObject::create,
3761 1 : "LexicalEnvironmentObject::create");
3762 :
3763 : void
3764 0 : CodeGenerator::visitNewLexicalEnvironmentObject(LNewLexicalEnvironmentObject* lir)
3765 : {
3766 0 : pushArg(Imm32(gc::DefaultHeap));
3767 0 : pushArg(ToRegister(lir->enclosing()));
3768 0 : pushArg(ImmGCPtr(lir->mir()->scope()));
3769 0 : callVM(NewLexicalEnvironmentObjectInfo, lir);
3770 0 : }
3771 :
3772 : typedef JSObject* (*CopyLexicalEnvironmentObjectFn)(JSContext*, HandleObject, bool);
3773 0 : static const VMFunction CopyLexicalEnvironmentObjectInfo =
3774 0 : FunctionInfo<CopyLexicalEnvironmentObjectFn>(js::jit::CopyLexicalEnvironmentObject,
3775 1 : "js::jit::CopyLexicalEnvironmentObject");
3776 :
3777 : void
3778 0 : CodeGenerator::visitCopyLexicalEnvironmentObject(LCopyLexicalEnvironmentObject* lir)
3779 : {
3780 0 : pushArg(Imm32(lir->mir()->copySlots()));
3781 0 : pushArg(ToRegister(lir->env()));
3782 0 : callVM(CopyLexicalEnvironmentObjectInfo, lir);
3783 0 : }
3784 :
3785 : void
3786 56 : CodeGenerator::visitGuardShape(LGuardShape* guard)
3787 : {
3788 0 : Register obj = ToRegister(guard->input());
3789 0 : Register temp = ToTempRegisterOrInvalid(guard->temp());
3790 0 : Label bail;
3791 0 : masm.branchTestObjShape(Assembler::NotEqual, obj, guard->mir()->shape(), temp, obj, &bail);
3792 0 : bailoutFrom(&bail, guard->snapshot());
3793 56 : }
3794 :
3795 : void
3796 1 : CodeGenerator::visitGuardObjectGroup(LGuardObjectGroup* guard)
3797 : {
3798 0 : Register obj = ToRegister(guard->input());
3799 1 : Register temp = ToTempRegisterOrInvalid(guard->temp());
3800 : Assembler::Condition cond =
3801 0 : guard->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;
3802 0 : Label bail;
3803 0 : masm.branchTestObjGroup(cond, obj, guard->mir()->group(), temp, obj, &bail);
3804 0 : bailoutFrom(&bail, guard->snapshot());
3805 1 : }
3806 :
3807 : void
3808 2 : CodeGenerator::visitGuardObjectIdentity(LGuardObjectIdentity* guard)
3809 : {
3810 0 : Register input = ToRegister(guard->input());
3811 4 : Register expected = ToRegister(guard->expected());
3812 :
3813 : Assembler::Condition cond =
3814 0 : guard->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;
3815 0 : bailoutCmpPtr(cond, input, expected, guard->snapshot());
3816 2 : }
3817 :
3818 : void
3819 3 : CodeGenerator::visitGuardReceiverPolymorphic(LGuardReceiverPolymorphic* lir)
3820 : {
3821 0 : const MGuardReceiverPolymorphic* mir = lir->mir();
3822 0 : Register obj = ToRegister(lir->object());
3823 0 : Register temp1 = ToRegister(lir->temp1());
3824 6 : Register temp2 = ToRegister(lir->temp2());
3825 :
3826 6 : Label done;
3827 :
3828 0 : for (size_t i = 0; i < mir->numReceivers(); i++) {
3829 6 : const ReceiverGuard& receiver = mir->receiver(i);
3830 :
3831 0 : Label next;
3832 6 : GuardReceiver(masm, receiver, obj, temp1, temp2, &next, /* checkNullExpando = */ true);
3833 :
3834 0 : if (i == mir->numReceivers() - 1) {
3835 3 : bailoutFrom(&next, lir->snapshot());
3836 : } else {
3837 0 : masm.jump(&done);
3838 3 : masm.bind(&next);
3839 : }
3840 : }
3841 :
3842 0 : masm.bind(&done);
3843 3 : }
3844 :
3845 : void
3846 0 : CodeGenerator::visitGuardUnboxedExpando(LGuardUnboxedExpando* lir)
3847 : {
3848 0 : Label miss;
3849 :
3850 0 : Register obj = ToRegister(lir->object());
3851 0 : masm.branchPtr(lir->mir()->requireExpando() ? Assembler::Equal : Assembler::NotEqual,
3852 0 : Address(obj, UnboxedPlainObject::offsetOfExpando()), ImmWord(0), &miss);
3853 :
3854 0 : bailoutFrom(&miss, lir->snapshot());
3855 0 : }
3856 :
3857 : void
3858 0 : CodeGenerator::visitLoadUnboxedExpando(LLoadUnboxedExpando* lir)
3859 : {
3860 0 : Register obj = ToRegister(lir->object());
3861 0 : Register result = ToRegister(lir->getDef(0));
3862 :
3863 0 : masm.loadPtr(Address(obj, UnboxedPlainObject::offsetOfExpando()), result);
3864 0 : }
3865 :
3866 : void
3867 206 : CodeGenerator::visitTypeBarrierV(LTypeBarrierV* lir)
3868 : {
3869 0 : ValueOperand operand = ToValue(lir, LTypeBarrierV::Input);
3870 0 : Register unboxScratch = ToTempRegisterOrInvalid(lir->unboxTemp());
3871 206 : Register objScratch = ToTempRegisterOrInvalid(lir->objTemp());
3872 :
3873 : // guardObjectType may zero the payload/Value register on speculative paths
3874 : // (we should have a defineReuseInput allocation in this case).
3875 206 : Register spectreRegToZero = operand.payloadOrValueReg();
3876 :
3877 0 : Label miss;
3878 0 : masm.guardTypeSet(operand, lir->mir()->resultTypeSet(), lir->mir()->barrierKind(),
3879 0 : unboxScratch, objScratch, spectreRegToZero, &miss);
3880 0 : bailoutFrom(&miss, lir->snapshot());
3881 206 : }
3882 :
3883 : void
3884 125 : CodeGenerator::visitTypeBarrierO(LTypeBarrierO* lir)
3885 : {
3886 0 : Register obj = ToRegister(lir->object());
3887 0 : Register scratch = ToTempRegisterOrInvalid(lir->temp());
3888 375 : Label miss, ok;
3889 :
3890 0 : if (lir->mir()->type() == MIRType::ObjectOrNull) {
3891 0 : masm.comment("Object or Null");
3892 0 : Label* nullTarget = lir->mir()->resultTypeSet()->mightBeMIRType(MIRType::Null) ? &ok : &miss;
3893 0 : masm.branchTestPtr(Assembler::Zero, obj, obj, nullTarget);
3894 : } else {
3895 0 : MOZ_ASSERT(lir->mir()->type() == MIRType::Object);
3896 250 : MOZ_ASSERT(lir->mir()->barrierKind() != BarrierKind::TypeTagOnly);
3897 : }
3898 :
3899 0 : if (lir->mir()->barrierKind() != BarrierKind::TypeTagOnly) {
3900 125 : masm.comment("Type tag only");
3901 : // guardObjectType may zero the object register on speculative paths
3902 : // (we should have a defineReuseInput allocation in this case).
3903 0 : Register spectreRegToZero = obj;
3904 250 : masm.guardObjectType(obj, lir->mir()->resultTypeSet(), scratch, spectreRegToZero, &miss);
3905 : }
3906 :
3907 0 : bailoutFrom(&miss, lir->snapshot());
3908 0 : masm.bind(&ok);
3909 125 : }
3910 :
3911 : // Out-of-line path to update the store buffer.
3912 : class OutOfLineCallPostWriteBarrier : public OutOfLineCodeBase<CodeGenerator>
3913 : {
3914 : LInstruction* lir_;
3915 : const LAllocation* object_;
3916 :
3917 : public:
3918 : OutOfLineCallPostWriteBarrier(LInstruction* lir, const LAllocation* object)
3919 170 : : lir_(lir), object_(object)
3920 : { }
3921 :
3922 0 : void accept(CodeGenerator* codegen) override {
3923 0 : codegen->visitOutOfLineCallPostWriteBarrier(this);
3924 85 : }
3925 :
3926 : LInstruction* lir() const {
3927 : return lir_;
3928 : }
3929 : const LAllocation* object() const {
3930 : return object_;
3931 : }
3932 : };
3933 :
3934 : static void
3935 0 : EmitStoreBufferCheckForConstant(MacroAssembler& masm, const gc::TenuredCell* cell,
3936 : AllocatableGeneralRegisterSet& regs, Label* exit, Label* callVM)
3937 : {
3938 0 : Register temp = regs.takeAny();
3939 :
3940 0 : gc::Arena* arena = cell->arena();
3941 :
3942 0 : Register cells = temp;
3943 0 : masm.loadPtr(AbsoluteAddress(&arena->bufferedCells()), cells);
3944 :
3945 0 : size_t index = gc::ArenaCellSet::getCellIndex(cell);
3946 : size_t word;
3947 : uint32_t mask;
3948 0 : gc::ArenaCellSet::getWordIndexAndMask(index, &word, &mask);
3949 0 : size_t offset = gc::ArenaCellSet::offsetOfBits() + word * sizeof(uint32_t);
3950 :
3951 0 : masm.branchTest32(Assembler::NonZero, Address(cells, offset), Imm32(mask), exit);
3952 :
3953 : // Check whether this is the sentinel set and if so call the VM to allocate
3954 : // one for this arena.
3955 0 : masm.branchPtr(Assembler::Equal, Address(cells, gc::ArenaCellSet::offsetOfArena()),
3956 0 : ImmPtr(nullptr), callVM);
3957 :
3958 : // Add the cell to the set.
3959 0 : masm.or32(Imm32(mask), Address(cells, offset));
3960 0 : masm.jump(exit);
3961 :
3962 0 : regs.add(temp);
3963 0 : }
3964 :
3965 : static void
3966 97 : EmitPostWriteBarrier(MacroAssembler& masm, CompileRuntime* runtime, Register objreg,
3967 : JSObject* maybeConstant, bool isGlobal, AllocatableGeneralRegisterSet& regs)
3968 : {
3969 97 : MOZ_ASSERT_IF(isGlobal, maybeConstant);
3970 :
3971 0 : Label callVM;
3972 194 : Label exit;
3973 :
3974 : // We already have a fast path to check whether a global is in the store
3975 : // buffer.
3976 0 : if (!isGlobal && maybeConstant)
3977 0 : EmitStoreBufferCheckForConstant(masm, &maybeConstant->asTenured(), regs, &exit, &callVM);
3978 :
3979 : // Call into the VM to barrier the write.
3980 97 : masm.bind(&callVM);
3981 :
3982 0 : Register runtimereg = regs.takeAny();
3983 194 : masm.mov(ImmPtr(runtime), runtimereg);
3984 :
3985 0 : masm.setupUnalignedABICall(regs.takeAny());
3986 0 : masm.passABIArg(runtimereg);
3987 0 : masm.passABIArg(objreg);
3988 0 : if (isGlobal)
3989 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, PostGlobalWriteBarrier));
3990 : else
3991 97 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, PostWriteBarrier));
3992 :
3993 0 : masm.bind(&exit);
3994 97 : }
3995 :
3996 : void
3997 85 : CodeGenerator::emitPostWriteBarrier(const LAllocation* obj)
3998 : {
3999 170 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
4000 :
4001 : Register objreg;
4002 0 : JSObject* object = nullptr;
4003 0 : bool isGlobal = false;
4004 0 : if (obj->isConstant()) {
4005 0 : object = &obj->toConstant()->toObject();
4006 0 : isGlobal = isGlobalObject(object);
4007 0 : objreg = regs.takeAny();
4008 0 : masm.movePtr(ImmGCPtr(object), objreg);
4009 : } else {
4010 85 : objreg = ToRegister(obj);
4011 : regs.takeUnchecked(objreg);
4012 : }
4013 :
4014 0 : EmitPostWriteBarrier(masm, gen->runtime, objreg, object, isGlobal, regs);
4015 85 : }
4016 :
4017 : void
4018 12 : CodeGenerator::emitPostWriteBarrier(Register objreg)
4019 : {
4020 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
4021 0 : regs.takeUnchecked(objreg);
4022 0 : EmitPostWriteBarrier(masm, gen->runtime, objreg, nullptr, false, regs);
4023 12 : }
4024 :
4025 : void
4026 85 : CodeGenerator::visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool)
4027 : {
4028 0 : saveLiveVolatile(ool->lir());
4029 0 : const LAllocation* obj = ool->object();
4030 0 : emitPostWriteBarrier(obj);
4031 85 : restoreLiveVolatile(ool->lir());
4032 :
4033 0 : masm.jump(ool->rejoin());
4034 85 : }
4035 :
4036 : void
4037 94 : CodeGenerator::maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal, OutOfLineCode* ool)
4038 : {
4039 : // Check whether an object is a global that we have already barriered before
4040 : // calling into the VM.
4041 : //
4042 : // We only check for the script's global, not other globals within the same
4043 : // compartment, because we bake in a pointer to realm->globalWriteBarriered
4044 : // and doing that would be invalid for other realms because they could be
4045 : // collected before the Ion code is discarded.
4046 :
4047 0 : if (!maybeGlobal->isConstant())
4048 94 : return;
4049 :
4050 0 : JSObject* obj = &maybeGlobal->toConstant()->toObject();
4051 0 : if (gen->realm->maybeGlobal() != obj)
4052 : return;
4053 :
4054 0 : auto addr = AbsoluteAddress(gen->realm->addressOfGlobalWriteBarriered());
4055 0 : masm.branch32(Assembler::NotEqual, addr, Imm32(0), ool->rejoin());
4056 : }
4057 :
4058 : template <class LPostBarrierType, MIRType nurseryType>
4059 : void
4060 57 : CodeGenerator::visitPostWriteBarrierCommon(LPostBarrierType* lir, OutOfLineCode* ool)
4061 : {
4062 114 : addOutOfLineCode(ool, lir->mir());
4063 :
4064 57 : Register temp = ToTempRegisterOrInvalid(lir->temp());
4065 :
4066 114 : if (lir->object()->isConstant()) {
4067 : // Constant nursery objects cannot appear here, see
4068 : // LIRGenerator::visitPostWriteElementBarrier.
4069 0 : MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()));
4070 : } else {
4071 171 : masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()), temp,
4072 : ool->rejoin());
4073 : }
4074 :
4075 57 : maybeEmitGlobalBarrierCheck(lir->object(), ool);
4076 :
4077 114 : Register value = ToRegister(lir->value());
4078 : if (nurseryType == MIRType::Object) {
4079 0 : if (lir->mir()->value()->type() == MIRType::ObjectOrNull)
4080 0 : masm.branchTestPtr(Assembler::Zero, value, value, ool->rejoin());
4081 : else
4082 153 : MOZ_ASSERT(lir->mir()->value()->type() == MIRType::Object);
4083 : } else {
4084 : MOZ_ASSERT(nurseryType == MIRType::String);
4085 18 : MOZ_ASSERT(lir->mir()->value()->type() == MIRType::String);
4086 : }
4087 57 : masm.branchPtrInNurseryChunk(Assembler::Equal, value, temp, ool->entry());
4088 :
4089 0 : masm.bind(ool->rejoin());
4090 57 : }
4091 :
4092 : template <class LPostBarrierType>
4093 : void
4094 37 : CodeGenerator::visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode* ool)
4095 : {
4096 74 : addOutOfLineCode(ool, lir->mir());
4097 :
4098 37 : Register temp = ToTempRegisterOrInvalid(lir->temp());
4099 :
4100 74 : if (lir->object()->isConstant()) {
4101 : // Constant nursery objects cannot appear here, see LIRGenerator::visitPostWriteElementBarrier.
4102 0 : MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()));
4103 : } else {
4104 111 : masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()), temp,
4105 : ool->rejoin());
4106 : }
4107 :
4108 37 : maybeEmitGlobalBarrierCheck(lir->object(), ool);
4109 :
4110 37 : ValueOperand value = ToValue(lir, LPostBarrierType::Input);
4111 : // Bug 1386094 - most callers only need to check for object or string, not
4112 : // both.
4113 37 : masm.branchValueIsNurseryCell(Assembler::Equal, value, temp, ool->entry());
4114 :
4115 0 : masm.bind(ool->rejoin());
4116 37 : }
4117 :
4118 : void
4119 44 : CodeGenerator::visitPostWriteBarrierO(LPostWriteBarrierO* lir)
4120 : {
4121 0 : auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
4122 0 : visitPostWriteBarrierCommon<LPostWriteBarrierO, MIRType::Object>(lir, ool);
4123 44 : }
4124 :
4125 : void
4126 6 : CodeGenerator::visitPostWriteBarrierS(LPostWriteBarrierS* lir)
4127 : {
4128 0 : auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
4129 0 : visitPostWriteBarrierCommon<LPostWriteBarrierS, MIRType::String>(lir, ool);
4130 6 : }
4131 :
4132 : void
4133 35 : CodeGenerator::visitPostWriteBarrierV(LPostWriteBarrierV* lir)
4134 : {
4135 0 : auto ool = new(alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
4136 0 : visitPostWriteBarrierCommonV(lir, ool);
4137 35 : }
4138 :
4139 : // Out-of-line path to update the store buffer.
4140 : class OutOfLineCallPostWriteElementBarrier : public OutOfLineCodeBase<CodeGenerator>
4141 : {
4142 : LInstruction* lir_;
4143 : const LAllocation* object_;
4144 : const LAllocation* index_;
4145 :
4146 : public:
4147 : OutOfLineCallPostWriteElementBarrier(LInstruction* lir, const LAllocation* object,
4148 : const LAllocation* index)
4149 9 : : lir_(lir),
4150 : object_(object),
4151 18 : index_(index)
4152 : { }
4153 :
4154 0 : void accept(CodeGenerator* codegen) override {
4155 0 : codegen->visitOutOfLineCallPostWriteElementBarrier(this);
4156 9 : }
4157 :
4158 : LInstruction* lir() const {
4159 : return lir_;
4160 : }
4161 :
4162 : const LAllocation* object() const {
4163 : return object_;
4164 : }
4165 :
4166 : const LAllocation* index() const {
4167 : return index_;
4168 : }
4169 : };
4170 :
4171 : void
4172 9 : CodeGenerator::visitOutOfLineCallPostWriteElementBarrier(OutOfLineCallPostWriteElementBarrier* ool)
4173 : {
4174 9 : saveLiveVolatile(ool->lir());
4175 :
4176 0 : const LAllocation* obj = ool->object();
4177 9 : const LAllocation* index = ool->index();
4178 :
4179 0 : Register objreg = obj->isConstant() ? InvalidReg : ToRegister(obj);
4180 9 : Register indexreg = ToRegister(index);
4181 :
4182 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
4183 9 : regs.takeUnchecked(indexreg);
4184 :
4185 0 : if (obj->isConstant()) {
4186 0 : objreg = regs.takeAny();
4187 0 : masm.movePtr(ImmGCPtr(&obj->toConstant()->toObject()), objreg);
4188 : } else {
4189 : regs.takeUnchecked(objreg);
4190 : }
4191 :
4192 0 : Register runtimereg = regs.takeAny();
4193 0 : masm.setupUnalignedABICall(runtimereg);
4194 0 : masm.mov(ImmPtr(gen->runtime), runtimereg);
4195 0 : masm.passABIArg(runtimereg);
4196 0 : masm.passABIArg(objreg);
4197 0 : masm.passABIArg(indexreg);
4198 9 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, (PostWriteElementBarrier<IndexInBounds::Maybe>)));
4199 :
4200 9 : restoreLiveVolatile(ool->lir());
4201 :
4202 0 : masm.jump(ool->rejoin());
4203 9 : }
4204 :
4205 : void
4206 7 : CodeGenerator::visitPostWriteElementBarrierO(LPostWriteElementBarrierO* lir)
4207 : {
4208 0 : auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
4209 0 : visitPostWriteBarrierCommon<LPostWriteElementBarrierO, MIRType::Object>(lir, ool);
4210 7 : }
4211 :
4212 : void
4213 0 : CodeGenerator::visitPostWriteElementBarrierS(LPostWriteElementBarrierS* lir)
4214 : {
4215 0 : auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
4216 0 : visitPostWriteBarrierCommon<LPostWriteElementBarrierS, MIRType::String>(lir, ool);
4217 0 : }
4218 :
4219 : void
4220 2 : CodeGenerator::visitPostWriteElementBarrierV(LPostWriteElementBarrierV* lir)
4221 : {
4222 0 : auto ool = new(alloc()) OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
4223 0 : visitPostWriteBarrierCommonV(lir, ool);
4224 2 : }
4225 :
4226 : void
4227 81 : CodeGenerator::visitCallNative(LCallNative* call)
4228 : {
4229 0 : WrappedFunction* target = call->getSingleTarget();
4230 0 : MOZ_ASSERT(target);
4231 162 : MOZ_ASSERT(target->isNativeWithCppEntry());
4232 :
4233 0 : int callargslot = call->argslot();
4234 81 : int unusedStack = StackOffsetOfPassedArg(callargslot);
4235 :
4236 : // Registers used for callWithABI() argument-passing.
4237 0 : const Register argContextReg = ToRegister(call->getArgContextReg());
4238 0 : const Register argUintNReg = ToRegister(call->getArgUintNReg());
4239 162 : const Register argVpReg = ToRegister(call->getArgVpReg());
4240 :
4241 : // Misc. temporary registers.
4242 162 : const Register tempReg = ToRegister(call->getTempReg());
4243 :
4244 243 : DebugOnly<uint32_t> initialStack = masm.framePushed();
4245 :
4246 81 : masm.checkStackAlignment();
4247 :
4248 : // Native functions have the signature:
4249 : // bool (*)(JSContext*, unsigned, Value* vp)
4250 : // Where vp[0] is space for an outparam, vp[1] is |this|, and vp[2] onward
4251 : // are the function arguments.
4252 :
4253 : // Allocate space for the outparam, moving the StackPointer to what will be &vp[1].
4254 81 : masm.adjustStack(unusedStack);
4255 :
4256 : // Push a Value containing the callee object: natives are allowed to access
4257 : // their callee before setting the return value. The StackPointer is moved
4258 : // to &vp[0].
4259 243 : masm.Push(ObjectValue(*target->rawJSFunction()));
4260 :
4261 : // Preload arguments into registers.
4262 0 : masm.loadJSContext(argContextReg);
4263 0 : masm.move32(Imm32(call->numActualArgs()), argUintNReg);
4264 162 : masm.moveStackPtrTo(argVpReg);
4265 :
4266 81 : masm.Push(argUintNReg);
4267 :
4268 : // Construct native exit frame.
4269 0 : uint32_t safepointOffset = masm.buildFakeExitFrame(tempReg);
4270 243 : masm.enterFakeExitFrameForNative(argContextReg, tempReg, call->mir()->isConstructing());
4271 :
4272 81 : markSafepointAt(safepointOffset, call);
4273 :
4274 162 : emitTracelogStartEvent(TraceLogger_Call);
4275 :
4276 : // Construct and execute call.
4277 0 : masm.setupUnalignedABICall(tempReg);
4278 0 : masm.passABIArg(argContextReg);
4279 0 : masm.passABIArg(argUintNReg);
4280 0 : masm.passABIArg(argVpReg);
4281 0 : JSNative native = target->native();
4282 0 : if (call->ignoresReturnValue() && target->hasJitInfo()) {
4283 0 : const JSJitInfo* jitInfo = target->jitInfo();
4284 0 : if (jitInfo->type() == JSJitInfo::IgnoresReturnValueNative)
4285 0 : native = jitInfo->ignoresReturnValueMethod;
4286 : }
4287 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, native), MoveOp::GENERAL,
4288 81 : CheckUnsafeCallWithABI::DontCheckHasExitFrame);
4289 :
4290 162 : emitTracelogStopEvent(TraceLogger_Call);
4291 :
4292 : // Test for failure.
4293 243 : masm.branchIfFalseBool(ReturnReg, masm.failureLabel());
4294 :
4295 : // Load the outparam vp[0] into output register(s).
4296 162 : masm.loadValue(Address(masm.getStackPointer(), NativeExitFrameLayout::offsetOfResult()), JSReturnOperand);
4297 :
4298 : // Until C++ code is instrumented against Spectre, prevent speculative
4299 : // execution from returning any private data.
4300 0 : if (JitOptions.spectreJitToCxxCalls && !call->mir()->ignoresReturnValue() &&
4301 154 : call->mir()->hasLiveDefUses())
4302 : {
4303 77 : masm.speculationBarrier();
4304 : }
4305 :
4306 : // The next instruction is removing the footer of the exit frame, so there
4307 : // is no need for leaveFakeExitFrame.
4308 :
4309 : // Move the StackPointer back to its original location, unwinding the native exit frame.
4310 0 : masm.adjustStack(NativeExitFrameLayout::Size() - unusedStack);
4311 0 : MOZ_ASSERT(masm.framePushed() == initialStack);
4312 81 : }
4313 :
4314 : static void
4315 0 : LoadDOMPrivate(MacroAssembler& masm, Register obj, Register priv, DOMObjectKind kind)
4316 : {
4317 : // Load the value in DOM_OBJECT_SLOT for a native or proxy DOM object. This
4318 : // will be in the first slot but may be fixed or non-fixed.
4319 0 : MOZ_ASSERT(obj != priv);
4320 :
4321 : // Check if it's a proxy.
4322 0 : Label isProxy, done;
4323 0 : if (kind == DOMObjectKind::Unknown)
4324 0 : masm.branchTestObjectIsProxy(true, obj, priv, &isProxy);
4325 :
4326 0 : if (kind != DOMObjectKind::Proxy) {
4327 : // If it's a native object, the value must be in a fixed slot.
4328 0 : masm.debugAssertObjHasFixedSlots(obj, priv);
4329 0 : masm.loadPrivate(Address(obj, NativeObject::getFixedSlotOffset(0)), priv);
4330 0 : if (kind == DOMObjectKind::Unknown)
4331 0 : masm.jump(&done);
4332 : }
4333 :
4334 0 : if (kind != DOMObjectKind::Native) {
4335 0 : masm.bind(&isProxy);
4336 : #ifdef DEBUG
4337 : // Sanity check: it must be a DOM proxy.
4338 0 : Label isDOMProxy;
4339 0 : masm.branchTestProxyHandlerFamily(Assembler::Equal, obj, priv,
4340 0 : GetDOMProxyHandlerFamily(), &isDOMProxy);
4341 0 : masm.assumeUnreachable("Expected a DOM proxy");
4342 0 : masm.bind(&isDOMProxy);
4343 : #endif
4344 0 : masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), priv);
4345 0 : masm.loadPrivate(Address(priv, detail::ProxyReservedSlots::offsetOfSlot(0)), priv);
4346 : }
4347 :
4348 0 : masm.bind(&done);
4349 0 : }
4350 :
4351 : void
4352 0 : CodeGenerator::visitCallDOMNative(LCallDOMNative* call)
4353 : {
4354 0 : WrappedFunction* target = call->getSingleTarget();
4355 0 : MOZ_ASSERT(target);
4356 0 : MOZ_ASSERT(target->isNative());
4357 0 : MOZ_ASSERT(target->hasJitInfo());
4358 0 : MOZ_ASSERT(call->mir()->isCallDOMNative());
4359 :
4360 0 : int callargslot = call->argslot();
4361 0 : int unusedStack = StackOffsetOfPassedArg(callargslot);
4362 :
4363 : // Registers used for callWithABI() argument-passing.
4364 0 : const Register argJSContext = ToRegister(call->getArgJSContext());
4365 0 : const Register argObj = ToRegister(call->getArgObj());
4366 0 : const Register argPrivate = ToRegister(call->getArgPrivate());
4367 0 : const Register argArgs = ToRegister(call->getArgArgs());
4368 :
4369 0 : DebugOnly<uint32_t> initialStack = masm.framePushed();
4370 :
4371 0 : masm.checkStackAlignment();
4372 :
4373 : // DOM methods have the signature:
4374 : // bool (*)(JSContext*, HandleObject, void* private, const JSJitMethodCallArgs& args)
4375 : // Where args is initialized from an argc and a vp, vp[0] is space for an
4376 : // outparam and the callee, vp[1] is |this|, and vp[2] onward are the
4377 : // function arguments. Note that args stores the argv, not the vp, and
4378 : // argv == vp + 2.
4379 :
4380 : // Nestle the stack up against the pushed arguments, leaving StackPointer at
4381 : // &vp[1]
4382 0 : masm.adjustStack(unusedStack);
4383 : // argObj is filled with the extracted object, then returned.
4384 0 : Register obj = masm.extractObject(Address(masm.getStackPointer(), 0), argObj);
4385 0 : MOZ_ASSERT(obj == argObj);
4386 :
4387 : // Push a Value containing the callee object: natives are allowed to access their callee before
4388 : // setitng the return value. After this the StackPointer points to &vp[0].
4389 0 : masm.Push(ObjectValue(*target->rawJSFunction()));
4390 :
4391 : // Now compute the argv value. Since StackPointer is pointing to &vp[0] and
4392 : // argv is &vp[2] we just need to add 2*sizeof(Value) to the current
4393 : // StackPointer.
4394 : JS_STATIC_ASSERT(JSJitMethodCallArgsTraits::offsetOfArgv == 0);
4395 : JS_STATIC_ASSERT(JSJitMethodCallArgsTraits::offsetOfArgc ==
4396 : IonDOMMethodExitFrameLayoutTraits::offsetOfArgcFromArgv);
4397 0 : masm.computeEffectiveAddress(Address(masm.getStackPointer(), 2 * sizeof(Value)), argArgs);
4398 :
4399 0 : LoadDOMPrivate(masm, obj, argPrivate, static_cast<MCallDOMNative*>(call->mir())->objectKind());
4400 :
4401 : // Push argc from the call instruction into what will become the IonExitFrame
4402 0 : masm.Push(Imm32(call->numActualArgs()));
4403 :
4404 : // Push our argv onto the stack
4405 0 : masm.Push(argArgs);
4406 : // And store our JSJitMethodCallArgs* in argArgs.
4407 0 : masm.moveStackPtrTo(argArgs);
4408 :
4409 : // Push |this| object for passing HandleObject. We push after argc to
4410 : // maintain the same sp-relative location of the object pointer with other
4411 : // DOMExitFrames.
4412 0 : masm.Push(argObj);
4413 0 : masm.moveStackPtrTo(argObj);
4414 :
4415 : // Construct native exit frame.
4416 0 : uint32_t safepointOffset = masm.buildFakeExitFrame(argJSContext);
4417 0 : masm.loadJSContext(argJSContext);
4418 0 : masm.enterFakeExitFrame(argJSContext, argJSContext, ExitFrameType::IonDOMMethod);
4419 :
4420 0 : markSafepointAt(safepointOffset, call);
4421 :
4422 : // Construct and execute call.
4423 0 : masm.setupUnalignedABICall(argJSContext);
4424 0 : masm.loadJSContext(argJSContext);
4425 0 : masm.passABIArg(argJSContext);
4426 0 : masm.passABIArg(argObj);
4427 0 : masm.passABIArg(argPrivate);
4428 0 : masm.passABIArg(argArgs);
4429 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, target->jitInfo()->method), MoveOp::GENERAL,
4430 0 : CheckUnsafeCallWithABI::DontCheckHasExitFrame);
4431 :
4432 0 : if (target->jitInfo()->isInfallible) {
4433 0 : masm.loadValue(Address(masm.getStackPointer(), IonDOMMethodExitFrameLayout::offsetOfResult()),
4434 0 : JSReturnOperand);
4435 : } else {
4436 : // Test for failure.
4437 0 : masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
4438 :
4439 : // Load the outparam vp[0] into output register(s).
4440 0 : masm.loadValue(Address(masm.getStackPointer(), IonDOMMethodExitFrameLayout::offsetOfResult()),
4441 0 : JSReturnOperand);
4442 : }
4443 :
4444 : // Until C++ code is instrumented against Spectre, prevent speculative
4445 : // execution from returning any private data.
4446 0 : if (JitOptions.spectreJitToCxxCalls && call->mir()->hasLiveDefUses())
4447 0 : masm.speculationBarrier();
4448 :
4449 : // The next instruction is removing the footer of the exit frame, so there
4450 : // is no need for leaveFakeExitFrame.
4451 :
4452 : // Move the StackPointer back to its original location, unwinding the native exit frame.
4453 0 : masm.adjustStack(IonDOMMethodExitFrameLayout::Size() - unusedStack);
4454 0 : MOZ_ASSERT(masm.framePushed() == initialStack);
4455 0 : }
4456 :
4457 : typedef bool (*GetIntrinsicValueFn)(JSContext* cx, HandlePropertyName, MutableHandleValue);
4458 0 : static const VMFunction GetIntrinsicValueInfo =
4459 3 : FunctionInfo<GetIntrinsicValueFn>(GetIntrinsicValue, "GetIntrinsicValue");
4460 :
4461 : void
4462 68 : CodeGenerator::visitCallGetIntrinsicValue(LCallGetIntrinsicValue* lir)
4463 : {
4464 0 : pushArg(ImmGCPtr(lir->mir()->name()));
4465 0 : callVM(GetIntrinsicValueInfo, lir);
4466 68 : }
4467 :
4468 : typedef bool (*InvokeFunctionFn)(JSContext*, HandleObject, bool, bool, uint32_t, Value*,
4469 : MutableHandleValue);
4470 0 : static const VMFunction InvokeFunctionInfo =
4471 3 : FunctionInfo<InvokeFunctionFn>(InvokeFunction, "InvokeFunction");
4472 :
4473 : void
4474 133 : CodeGenerator::emitCallInvokeFunction(LInstruction* call, Register calleereg,
4475 : bool constructing, bool ignoresReturnValue,
4476 : uint32_t argc, uint32_t unusedStack)
4477 : {
4478 : // Nestle %esp up to the argument vector.
4479 : // Each path must account for framePushed_ separately, for callVM to be valid.
4480 133 : masm.freeStack(unusedStack);
4481 :
4482 0 : pushArg(masm.getStackPointer()); // argv.
4483 0 : pushArg(Imm32(argc)); // argc.
4484 0 : pushArg(Imm32(ignoresReturnValue));
4485 0 : pushArg(Imm32(constructing)); // constructing.
4486 266 : pushArg(calleereg); // JSFunction*.
4487 :
4488 133 : callVM(InvokeFunctionInfo, call);
4489 :
4490 : // Un-nestle %esp from the argument vector. No prefix was pushed.
4491 0 : masm.reserveStack(unusedStack);
4492 133 : }
4493 :
4494 : void
4495 113 : CodeGenerator::visitCallGeneric(LCallGeneric* call)
4496 : {
4497 0 : Register calleereg = ToRegister(call->getFunction());
4498 0 : Register objreg = ToRegister(call->getTempObject());
4499 0 : Register nargsreg = ToRegister(call->getNargsReg());
4500 0 : uint32_t unusedStack = StackOffsetOfPassedArg(call->argslot());
4501 565 : Label invoke, thunk, makeCall, end;
4502 :
4503 : // Known-target case is handled by LCallKnown.
4504 226 : MOZ_ASSERT(!call->hasSingleTarget());
4505 :
4506 113 : masm.checkStackAlignment();
4507 :
4508 : // Guard that calleereg is actually a function object.
4509 0 : if (call->mir()->needsClassCheck()) {
4510 0 : masm.branchTestObjClass(Assembler::NotEqual, calleereg, &JSFunction::class_, nargsreg,
4511 112 : calleereg, &invoke);
4512 : }
4513 :
4514 : // Guard that calleereg is an interpreted function with a JSScript or a
4515 : // wasm function.
4516 : // If we are constructing, also ensure the callee is a constructor.
4517 0 : if (call->mir()->isConstructing()) {
4518 2 : masm.branchIfNotInterpretedConstructor(calleereg, nargsreg, &invoke);
4519 : } else {
4520 0 : masm.branchIfFunctionHasNoJitEntry(calleereg, /* isConstructing */ false, &invoke);
4521 0 : masm.branchFunctionKind(Assembler::Equal, JSFunction::ClassConstructor, calleereg, objreg,
4522 111 : &invoke);
4523 : }
4524 :
4525 0 : if (call->mir()->needsArgCheck())
4526 113 : masm.loadJitCodeRaw(calleereg, objreg);
4527 : else
4528 0 : masm.loadJitCodeNoArgCheck(calleereg, objreg);
4529 :
4530 : // Nestle the StackPointer up to the argument vector.
4531 113 : masm.freeStack(unusedStack);
4532 :
4533 : // Construct the IonFramePrefix.
4534 0 : uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS,
4535 0 : JitFrameLayout::Size());
4536 0 : masm.Push(Imm32(call->numActualArgs()));
4537 0 : masm.PushCalleeToken(calleereg, call->mir()->isConstructing());
4538 226 : masm.Push(Imm32(descriptor));
4539 :
4540 : // Check whether the provided arguments satisfy target argc.
4541 : // We cannot have lowered to LCallGeneric with a known target. Assert that we didn't
4542 : // add any undefineds in IonBuilder. NB: MCall::numStackArgs includes |this|.
4543 0 : DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing();
4544 0 : MOZ_ASSERT(call->numActualArgs() == call->mir()->numStackArgs() - numNonArgsOnStack);
4545 0 : masm.load16ZeroExtend(Address(calleereg, JSFunction::offsetOfNargs()), nargsreg);
4546 0 : masm.branch32(Assembler::Above, nargsreg, Imm32(call->numActualArgs()), &thunk);
4547 226 : masm.jump(&makeCall);
4548 :
4549 : // Argument fixup needed. Load the ArgumentsRectifier.
4550 113 : masm.bind(&thunk);
4551 : {
4552 0 : TrampolinePtr argumentsRectifier = gen->jitRuntime()->getArgumentsRectifier();
4553 113 : masm.movePtr(argumentsRectifier, objreg);
4554 : }
4555 :
4556 : // Finally call the function in objreg.
4557 0 : masm.bind(&makeCall);
4558 0 : uint32_t callOffset = masm.callJit(objreg);
4559 113 : markSafepointAt(callOffset, call);
4560 :
4561 : // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.
4562 : // The return address has already been removed from the Ion frame.
4563 0 : int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*);
4564 0 : masm.adjustStack(prefixGarbage - unusedStack);
4565 226 : masm.jump(&end);
4566 :
4567 : // Handle uncompiled or native functions.
4568 0 : masm.bind(&invoke);
4569 0 : emitCallInvokeFunction(call, calleereg, call->isConstructing(), call->ignoresReturnValue(),
4570 113 : call->numActualArgs(), unusedStack);
4571 :
4572 113 : masm.bind(&end);
4573 :
4574 : // If the return value of the constructing function is Primitive,
4575 : // replace the return value with the Object from CreateThis.
4576 0 : if (call->mir()->isConstructing()) {
4577 0 : Label notPrimitive;
4578 0 : masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, ¬Primitive);
4579 0 : masm.loadValue(Address(masm.getStackPointer(), unusedStack), JSReturnOperand);
4580 2 : masm.bind(¬Primitive);
4581 : }
4582 113 : }
4583 :
4584 : typedef bool (*InvokeFunctionShuffleFn)(JSContext*, HandleObject, uint32_t, uint32_t, Value*,
4585 : MutableHandleValue);
4586 0 : static const VMFunction InvokeFunctionShuffleInfo =
4587 0 : FunctionInfo<InvokeFunctionShuffleFn>(InvokeFunctionShuffleNewTarget,
4588 1 : "InvokeFunctionShuffleNewTarget");
4589 : void
4590 0 : CodeGenerator::emitCallInvokeFunctionShuffleNewTarget(LCallKnown* call, Register calleeReg,
4591 : uint32_t numFormals, uint32_t unusedStack)
4592 : {
4593 0 : masm.freeStack(unusedStack);
4594 :
4595 0 : pushArg(masm.getStackPointer());
4596 0 : pushArg(Imm32(numFormals));
4597 0 : pushArg(Imm32(call->numActualArgs()));
4598 0 : pushArg(calleeReg);
4599 :
4600 0 : callVM(InvokeFunctionShuffleInfo, call);
4601 :
4602 0 : masm.reserveStack(unusedStack);
4603 0 : }
4604 :
4605 : void
4606 20 : CodeGenerator::visitCallKnown(LCallKnown* call)
4607 : {
4608 0 : Register calleereg = ToRegister(call->getFunction());
4609 0 : Register objreg = ToRegister(call->getTempObject());
4610 0 : uint32_t unusedStack = StackOffsetOfPassedArg(call->argslot());
4611 40 : WrappedFunction* target = call->getSingleTarget();
4612 :
4613 : // Native single targets (except wasm) are handled by LCallNative.
4614 20 : MOZ_ASSERT(!target->isNativeWithCppEntry());
4615 : // Missing arguments must have been explicitly appended by the IonBuilder.
4616 0 : DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing();
4617 60 : MOZ_ASSERT(target->nargs() <= call->mir()->numStackArgs() - numNonArgsOnStack);
4618 :
4619 40 : MOZ_ASSERT_IF(call->isConstructing(), target->isConstructor());
4620 :
4621 20 : masm.checkStackAlignment();
4622 :
4623 0 : if (target->isClassConstructor() && !call->isConstructing()) {
4624 0 : emitCallInvokeFunction(call, calleereg, call->isConstructing(), call->ignoresReturnValue(),
4625 0 : call->numActualArgs(), unusedStack);
4626 0 : return;
4627 : }
4628 :
4629 20 : MOZ_ASSERT_IF(target->isClassConstructor(), call->isConstructing());
4630 :
4631 0 : Label uncompiled;
4632 20 : if (!target->isNativeWithJitEntry()) {
4633 : // The calleereg is known to be a non-native function, but might point
4634 : // to a LazyScript instead of a JSScript.
4635 40 : masm.branchIfFunctionHasNoJitEntry(calleereg, call->isConstructing(), &uncompiled);
4636 : }
4637 :
4638 0 : if (call->mir()->needsArgCheck())
4639 15 : masm.loadJitCodeRaw(calleereg, objreg);
4640 : else
4641 5 : masm.loadJitCodeNoArgCheck(calleereg, objreg);
4642 :
4643 : // Nestle the StackPointer up to the argument vector.
4644 20 : masm.freeStack(unusedStack);
4645 :
4646 : // Construct the IonFramePrefix.
4647 0 : uint32_t descriptor = MakeFrameDescriptor(masm.framePushed(), JitFrame_IonJS,
4648 0 : JitFrameLayout::Size());
4649 0 : masm.Push(Imm32(call->numActualArgs()));
4650 0 : masm.PushCalleeToken(calleereg, call->mir()->isConstructing());
4651 40 : masm.Push(Imm32(descriptor));
4652 :
4653 : // Finally call the function in objreg.
4654 0 : uint32_t callOffset = masm.callJit(objreg);
4655 20 : markSafepointAt(callOffset, call);
4656 :
4657 : // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.
4658 : // The return address has already been removed from the Ion frame.
4659 0 : int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*);
4660 20 : masm.adjustStack(prefixGarbage - unusedStack);
4661 :
4662 0 : if (uncompiled.used()) {
4663 0 : Label end;
4664 40 : masm.jump(&end);
4665 :
4666 : // Handle uncompiled functions.
4667 0 : masm.bind(&uncompiled);
4668 0 : if (call->isConstructing() && target->nargs() > call->numActualArgs()) {
4669 0 : emitCallInvokeFunctionShuffleNewTarget(call, calleereg, target->nargs(), unusedStack);
4670 : } else {
4671 0 : emitCallInvokeFunction(call, calleereg, call->isConstructing(),
4672 60 : call->ignoresReturnValue(), call->numActualArgs(), unusedStack);
4673 : }
4674 :
4675 20 : masm.bind(&end);
4676 : }
4677 :
4678 : // If the return value of the constructing function is Primitive,
4679 : // replace the return value with the Object from CreateThis.
4680 0 : if (call->mir()->isConstructing()) {
4681 0 : Label notPrimitive;
4682 0 : masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand, ¬Primitive);
4683 0 : masm.loadValue(Address(masm.getStackPointer(), unusedStack), JSReturnOperand);
4684 0 : masm.bind(¬Primitive);
4685 : }
4686 : }
4687 :
4688 : template<typename T>
4689 : void
4690 0 : CodeGenerator::emitCallInvokeFunction(T* apply, Register extraStackSize)
4691 : {
4692 0 : Register objreg = ToRegister(apply->getTempObject());
4693 0 : MOZ_ASSERT(objreg != extraStackSize);
4694 :
4695 : // Push the space used by the arguments.
4696 0 : masm.moveStackPtrTo(objreg);
4697 0 : masm.Push(extraStackSize);
4698 :
4699 0 : pushArg(objreg); // argv.
4700 0 : pushArg(ToRegister(apply->getArgc())); // argc.
4701 0 : pushArg(Imm32(false)); // ignoresReturnValue.
4702 0 : pushArg(Imm32(false)); // isConstrucing.
4703 0 : pushArg(ToRegister(apply->getFunction())); // JSFunction*.
4704 :
4705 : // This specialization og callVM restore the extraStackSize after the call.
4706 0 : callVM(InvokeFunctionInfo, apply, &extraStackSize);
4707 :
4708 0 : masm.Pop(extraStackSize);
4709 0 : }
4710 :
4711 : // Do not bailout after the execution of this function since the stack no longer
4712 : // correspond to what is expected by the snapshots.
4713 : void
4714 0 : CodeGenerator::emitAllocateSpaceForApply(Register argcreg, Register extraStackSpace, Label* end)
4715 : {
4716 : // Initialize the loop counter AND Compute the stack usage (if == 0)
4717 0 : masm.movePtr(argcreg, extraStackSpace);
4718 :
4719 : // Align the JitFrameLayout on the JitStackAlignment.
4720 : if (JitStackValueAlignment > 1) {
4721 0 : MOZ_ASSERT(frameSize() % JitStackAlignment == 0,
4722 : "Stack padding assumes that the frameSize is correct");
4723 : MOZ_ASSERT(JitStackValueAlignment == 2);
4724 0 : Label noPaddingNeeded;
4725 : // if the number of arguments is odd, then we do not need any padding.
4726 0 : masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded);
4727 0 : masm.addPtr(Imm32(1), extraStackSpace);
4728 0 : masm.bind(&noPaddingNeeded);
4729 : }
4730 :
4731 : // Reserve space for copying the arguments.
4732 0 : NativeObject::elementsSizeMustNotOverflow();
4733 0 : masm.lshiftPtr(Imm32(ValueShift), extraStackSpace);
4734 0 : masm.subFromStackPtr(extraStackSpace);
4735 :
4736 : #ifdef DEBUG
4737 : // Put a magic value in the space reserved for padding. Note, this code
4738 : // cannot be merged with the previous test, as not all architectures can
4739 : // write below their stack pointers.
4740 : if (JitStackValueAlignment > 1) {
4741 : MOZ_ASSERT(JitStackValueAlignment == 2);
4742 0 : Label noPaddingNeeded;
4743 : // if the number of arguments is odd, then we do not need any padding.
4744 0 : masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded);
4745 0 : BaseValueIndex dstPtr(masm.getStackPointer(), argcreg);
4746 0 : masm.storeValue(MagicValue(JS_ARG_POISON), dstPtr);
4747 0 : masm.bind(&noPaddingNeeded);
4748 : }
4749 : #endif
4750 :
4751 : // Skip the copy of arguments if there are none.
4752 0 : masm.branchTestPtr(Assembler::Zero, argcreg, argcreg, end);
4753 0 : }
4754 :
4755 : // Destroys argvIndex and copyreg.
4756 : void
4757 0 : CodeGenerator::emitCopyValuesForApply(Register argvSrcBase, Register argvIndex, Register copyreg,
4758 : size_t argvSrcOffset, size_t argvDstOffset)
4759 : {
4760 0 : Label loop;
4761 0 : masm.bind(&loop);
4762 :
4763 : // As argvIndex is off by 1, and we use the decBranchPtr instruction
4764 : // to loop back, we have to substract the size of the word which are
4765 : // copied.
4766 0 : BaseValueIndex srcPtr(argvSrcBase, argvIndex, argvSrcOffset - sizeof(void*));
4767 0 : BaseValueIndex dstPtr(masm.getStackPointer(), argvIndex, argvDstOffset - sizeof(void*));
4768 0 : masm.loadPtr(srcPtr, copyreg);
4769 0 : masm.storePtr(copyreg, dstPtr);
4770 :
4771 : // Handle 32 bits architectures.
4772 : if (sizeof(Value) == 2 * sizeof(void*)) {
4773 : BaseValueIndex srcPtrLow(argvSrcBase, argvIndex, argvSrcOffset - 2 * sizeof(void*));
4774 : BaseValueIndex dstPtrLow(masm.getStackPointer(), argvIndex, argvDstOffset - 2 * sizeof(void*));
4775 : masm.loadPtr(srcPtrLow, copyreg);
4776 : masm.storePtr(copyreg, dstPtrLow);
4777 : }
4778 :
4779 0 : masm.decBranchPtr(Assembler::NonZero, argvIndex, Imm32(1), &loop);
4780 0 : }
4781 :
4782 : void
4783 0 : CodeGenerator::emitPopArguments(Register extraStackSpace)
4784 : {
4785 : // Pop |this| and Arguments.
4786 0 : masm.freeStack(extraStackSpace);
4787 0 : }
4788 :
4789 : void
4790 0 : CodeGenerator::emitPushArguments(LApplyArgsGeneric* apply, Register extraStackSpace)
4791 : {
4792 : // Holds the function nargs. Initially the number of args to the caller.
4793 0 : Register argcreg = ToRegister(apply->getArgc());
4794 0 : Register copyreg = ToRegister(apply->getTempObject());
4795 :
4796 0 : Label end;
4797 0 : emitAllocateSpaceForApply(argcreg, extraStackSpace, &end);
4798 :
4799 : // We are making a copy of the arguments which are above the JitFrameLayout
4800 : // of the current Ion frame.
4801 : //
4802 : // [arg1] [arg0] <- src [this] [JitFrameLayout] [.. frameSize ..] [pad] [arg1] [arg0] <- dst
4803 :
4804 : // Compute the source and destination offsets into the stack.
4805 0 : size_t argvSrcOffset = frameSize() + JitFrameLayout::offsetOfActualArgs();
4806 0 : size_t argvDstOffset = 0;
4807 :
4808 : // Save the extra stack space, and re-use the register as a base.
4809 0 : masm.push(extraStackSpace);
4810 0 : Register argvSrcBase = extraStackSpace;
4811 0 : argvSrcOffset += sizeof(void*);
4812 0 : argvDstOffset += sizeof(void*);
4813 :
4814 : // Save the actual number of register, and re-use the register as an index register.
4815 0 : masm.push(argcreg);
4816 0 : Register argvIndex = argcreg;
4817 0 : argvSrcOffset += sizeof(void*);
4818 0 : argvDstOffset += sizeof(void*);
4819 :
4820 : // srcPtr = (StackPointer + extraStackSpace) + argvSrcOffset
4821 : // dstPtr = (StackPointer ) + argvDstOffset
4822 0 : masm.addStackPtrTo(argvSrcBase);
4823 :
4824 : // Copy arguments.
4825 0 : emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, argvSrcOffset, argvDstOffset);
4826 :
4827 : // Restore argcreg and the extra stack space counter.
4828 0 : masm.pop(argcreg);
4829 0 : masm.pop(extraStackSpace);
4830 :
4831 : // Join with all arguments copied and the extra stack usage computed.
4832 0 : masm.bind(&end);
4833 :
4834 : // Push |this|.
4835 0 : masm.addPtr(Imm32(sizeof(Value)), extraStackSpace);
4836 0 : masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex));
4837 0 : }
4838 :
4839 : void
4840 0 : CodeGenerator::emitPushArguments(LApplyArrayGeneric* apply, Register extraStackSpace)
4841 : {
4842 0 : Label noCopy, epilogue;
4843 0 : Register tmpArgc = ToRegister(apply->getTempObject());
4844 0 : Register elementsAndArgc = ToRegister(apply->getElements());
4845 :
4846 : // Invariants guarded in the caller:
4847 : // - the array is not too long
4848 : // - the array length equals its initialized length
4849 :
4850 : // The array length is our argc for the purposes of allocating space.
4851 0 : Address length(ToRegister(apply->getElements()), ObjectElements::offsetOfLength());
4852 0 : masm.load32(length, tmpArgc);
4853 :
4854 : // Allocate space for the values.
4855 0 : emitAllocateSpaceForApply(tmpArgc, extraStackSpace, &noCopy);
4856 :
4857 : // Copy the values. This code is skipped entirely if there are
4858 : // no values.
4859 0 : size_t argvDstOffset = 0;
4860 :
4861 0 : Register argvSrcBase = elementsAndArgc; // Elements value
4862 :
4863 0 : masm.push(extraStackSpace);
4864 0 : Register copyreg = extraStackSpace;
4865 0 : argvDstOffset += sizeof(void*);
4866 :
4867 0 : masm.push(tmpArgc);
4868 0 : Register argvIndex = tmpArgc;
4869 0 : argvDstOffset += sizeof(void*);
4870 :
4871 : // Copy
4872 0 : emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, 0, argvDstOffset);
4873 :
4874 : // Restore.
4875 0 : masm.pop(elementsAndArgc);
4876 0 : masm.pop(extraStackSpace);
4877 0 : masm.jump(&epilogue);
4878 :
4879 : // Clear argc if we skipped the copy step.
4880 0 : masm.bind(&noCopy);
4881 0 : masm.movePtr(ImmPtr(0), elementsAndArgc);
4882 :
4883 : // Join with all arguments copied and the extra stack usage computed.
4884 : // Note, "elements" has become "argc".
4885 0 : masm.bind(&epilogue);
4886 :
4887 : // Push |this|.
4888 0 : masm.addPtr(Imm32(sizeof(Value)), extraStackSpace);
4889 0 : masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex));
4890 0 : }
4891 :
4892 : template<typename T>
4893 : void
4894 0 : CodeGenerator::emitApplyGeneric(T* apply)
4895 : {
4896 : // Holds the function object.
4897 0 : Register calleereg = ToRegister(apply->getFunction());
4898 :
4899 : // Temporary register for modifying the function object.
4900 0 : Register objreg = ToRegister(apply->getTempObject());
4901 0 : Register extraStackSpace = ToRegister(apply->getTempStackCounter());
4902 :
4903 : // Holds the function nargs, computed in the invoker or (for
4904 : // ApplyArray) in the argument pusher.
4905 0 : Register argcreg = ToRegister(apply->getArgc());
4906 :
4907 : // Unless already known, guard that calleereg is actually a function object.
4908 0 : if (!apply->hasSingleTarget()) {
4909 0 : Label bail;
4910 0 : masm.branchTestObjClass(Assembler::NotEqual, calleereg, &JSFunction::class_, objreg,
4911 : calleereg, &bail);
4912 0 : bailoutFrom(&bail, apply->snapshot());
4913 : }
4914 :
4915 : // Copy the arguments of the current function.
4916 : //
4917 : // In the case of ApplyArray, also compute argc: the argc register
4918 : // and the elements register are the same; argc must not be
4919 : // referenced before the call to emitPushArguments() and elements
4920 : // must not be referenced after it returns.
4921 : //
4922 : // objreg is dead across this call.
4923 : //
4924 : // extraStackSpace is garbage on entry and defined on exit.
4925 0 : emitPushArguments(apply, extraStackSpace);
4926 :
4927 0 : masm.checkStackAlignment();
4928 :
4929 : // If the function is native, only emit the call to InvokeFunction.
4930 0 : if (apply->hasSingleTarget() && apply->getSingleTarget()->isNativeWithCppEntry()) {
4931 0 : emitCallInvokeFunction(apply, extraStackSpace);
4932 0 : emitPopArguments(extraStackSpace);
4933 0 : return;
4934 : }
4935 :
4936 0 : Label end, invoke;
4937 :
4938 : // Guard that calleereg is an interpreted function with a JSScript.
4939 0 : masm.branchIfFunctionHasNoJitEntry(calleereg, /* constructing */ false, &invoke);
4940 :
4941 : // Guard that calleereg is not a class constrcuctor
4942 0 : masm.branchFunctionKind(Assembler::Equal, JSFunction::ClassConstructor,
4943 : calleereg, objreg, &invoke);
4944 :
4945 : // Knowing that calleereg is a non-native function, load jitcode.
4946 0 : masm.loadJitCodeRaw(calleereg, objreg);
4947 :
4948 : // Call with an Ion frame or a rectifier frame.
4949 : {
4950 : // Create the frame descriptor.
4951 0 : unsigned pushed = masm.framePushed();
4952 0 : Register stackSpace = extraStackSpace;
4953 0 : masm.addPtr(Imm32(pushed), stackSpace);
4954 0 : masm.makeFrameDescriptor(stackSpace, JitFrame_IonJS, JitFrameLayout::Size());
4955 :
4956 0 : masm.Push(argcreg);
4957 0 : masm.Push(calleereg);
4958 0 : masm.Push(stackSpace); // descriptor
4959 :
4960 0 : Label underflow, rejoin;
4961 :
4962 : // Check whether the provided arguments satisfy target argc.
4963 0 : if (!apply->hasSingleTarget()) {
4964 0 : Register nformals = extraStackSpace;
4965 0 : masm.load16ZeroExtend(Address(calleereg, JSFunction::offsetOfNargs()), nformals);
4966 0 : masm.branch32(Assembler::Below, argcreg, nformals, &underflow);
4967 : } else {
4968 0 : masm.branch32(Assembler::Below, argcreg, Imm32(apply->getSingleTarget()->nargs()),
4969 : &underflow);
4970 : }
4971 :
4972 : // Skip the construction of the rectifier frame because we have no
4973 : // underflow.
4974 0 : masm.jump(&rejoin);
4975 :
4976 : // Argument fixup needed. Get ready to call the argumentsRectifier.
4977 : {
4978 0 : masm.bind(&underflow);
4979 :
4980 : // Hardcode the address of the argumentsRectifier code.
4981 0 : TrampolinePtr argumentsRectifier = gen->jitRuntime()->getArgumentsRectifier();
4982 0 : masm.movePtr(argumentsRectifier, objreg);
4983 : }
4984 :
4985 0 : masm.bind(&rejoin);
4986 :
4987 : // Finally call the function in objreg, as assigned by one of the paths above.
4988 0 : uint32_t callOffset = masm.callJit(objreg);
4989 0 : markSafepointAt(callOffset, apply);
4990 :
4991 : // Recover the number of arguments from the frame descriptor.
4992 0 : masm.loadPtr(Address(masm.getStackPointer(), 0), stackSpace);
4993 0 : masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), stackSpace);
4994 0 : masm.subPtr(Imm32(pushed), stackSpace);
4995 :
4996 : // Increment to remove IonFramePrefix; decrement to fill FrameSizeClass.
4997 : // The return address has already been removed from the Ion frame.
4998 0 : int prefixGarbage = sizeof(JitFrameLayout) - sizeof(void*);
4999 0 : masm.adjustStack(prefixGarbage);
5000 0 : masm.jump(&end);
5001 : }
5002 :
5003 : // Handle uncompiled or native functions.
5004 : {
5005 0 : masm.bind(&invoke);
5006 0 : emitCallInvokeFunction(apply, extraStackSpace);
5007 : }
5008 :
5009 : // Pop arguments and continue.
5010 0 : masm.bind(&end);
5011 0 : emitPopArguments(extraStackSpace);
5012 : }
5013 :
5014 : void
5015 0 : CodeGenerator::visitApplyArgsGeneric(LApplyArgsGeneric* apply)
5016 : {
5017 : // Limit the number of parameters we can handle to a number that does not risk
5018 : // us allocating too much stack, notably on Windows where there is a 4K guard page
5019 : // that has to be touched to extend the stack. See bug 1351278. The value "3000"
5020 : // is the size of the guard page minus an arbitrary, but large, safety margin.
5021 :
5022 0 : LSnapshot* snapshot = apply->snapshot();
5023 0 : Register argcreg = ToRegister(apply->getArgc());
5024 :
5025 0 : uint32_t limit = 3000 / sizeof(Value);
5026 0 : bailoutCmp32(Assembler::Above, argcreg, Imm32(limit), snapshot);
5027 :
5028 0 : emitApplyGeneric(apply);
5029 0 : }
5030 :
5031 : void
5032 0 : CodeGenerator::visitApplyArrayGeneric(LApplyArrayGeneric* apply)
5033 : {
5034 0 : LSnapshot* snapshot = apply->snapshot();
5035 0 : Register tmp = ToRegister(apply->getTempObject());
5036 :
5037 0 : Address length(ToRegister(apply->getElements()), ObjectElements::offsetOfLength());
5038 0 : masm.load32(length, tmp);
5039 :
5040 : // See comment in visitApplyArgsGeneric, above.
5041 :
5042 0 : uint32_t limit = 3000 / sizeof(Value);
5043 0 : bailoutCmp32(Assembler::Above, tmp, Imm32(limit), snapshot);
5044 :
5045 : // Ensure that the array does not contain an uninitialized tail.
5046 :
5047 : Address initializedLength(ToRegister(apply->getElements()),
5048 0 : ObjectElements::offsetOfInitializedLength());
5049 0 : masm.sub32(initializedLength, tmp);
5050 0 : bailoutCmp32(Assembler::NotEqual, tmp, Imm32(0), snapshot);
5051 :
5052 0 : emitApplyGeneric(apply);
5053 0 : }
5054 :
5055 : void
5056 0 : CodeGenerator::visitBail(LBail* lir)
5057 : {
5058 1 : bailout(lir->snapshot());
5059 0 : }
5060 :
5061 : void
5062 0 : CodeGenerator::visitUnreachable(LUnreachable* lir)
5063 : {
5064 1 : masm.assumeUnreachable("end-of-block assumed unreachable");
5065 0 : }
5066 :
5067 : void
5068 0 : CodeGenerator::visitEncodeSnapshot(LEncodeSnapshot* lir)
5069 : {
5070 0 : encode(lir->snapshot());
5071 0 : }
5072 :
5073 : void
5074 0 : CodeGenerator::visitGetDynamicName(LGetDynamicName* lir)
5075 : {
5076 0 : Register envChain = ToRegister(lir->getEnvironmentChain());
5077 0 : Register name = ToRegister(lir->getName());
5078 0 : Register temp1 = ToRegister(lir->temp1());
5079 0 : Register temp2 = ToRegister(lir->temp2());
5080 0 : Register temp3 = ToRegister(lir->temp3());
5081 :
5082 0 : masm.loadJSContext(temp3);
5083 :
5084 : /* Make space for the outparam. */
5085 0 : masm.adjustStack(-int32_t(sizeof(Value)));
5086 0 : masm.moveStackPtrTo(temp2);
5087 :
5088 0 : masm.setupUnalignedABICall(temp1);
5089 0 : masm.passABIArg(temp3);
5090 0 : masm.passABIArg(envChain);
5091 0 : masm.passABIArg(name);
5092 0 : masm.passABIArg(temp2);
5093 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, GetDynamicName));
5094 :
5095 0 : const ValueOperand out = ToOutValue(lir);
5096 :
5097 0 : masm.loadValue(Address(masm.getStackPointer(), 0), out);
5098 0 : masm.adjustStack(sizeof(Value));
5099 :
5100 0 : Label undefined;
5101 0 : masm.branchTestUndefined(Assembler::Equal, out, &undefined);
5102 0 : bailoutFrom(&undefined, lir->snapshot());
5103 0 : }
5104 :
5105 : typedef bool (*DirectEvalSFn)(JSContext*, HandleObject, HandleScript, HandleValue,
5106 : HandleString, jsbytecode*, MutableHandleValue);
5107 0 : static const VMFunction DirectEvalStringInfo =
5108 3 : FunctionInfo<DirectEvalSFn>(DirectEvalStringFromIon, "DirectEvalStringFromIon");
5109 :
5110 : void
5111 0 : CodeGenerator::visitCallDirectEval(LCallDirectEval* lir)
5112 : {
5113 0 : Register envChain = ToRegister(lir->getEnvironmentChain());
5114 0 : Register string = ToRegister(lir->getString());
5115 :
5116 0 : pushArg(ImmPtr(lir->mir()->pc()));
5117 0 : pushArg(string);
5118 0 : pushArg(ToValue(lir, LCallDirectEval::NewTarget));
5119 0 : pushArg(ImmGCPtr(current->mir()->info().script()));
5120 0 : pushArg(envChain);
5121 :
5122 0 : callVM(DirectEvalStringInfo, lir);
5123 0 : }
5124 :
5125 : void
5126 96 : CodeGenerator::generateArgumentsChecks(bool assert)
5127 : {
5128 : // This function can be used the normal way to check the argument types,
5129 : // before entering the function and bailout when arguments don't match.
5130 : // For debug purpose, this is can also be used to force/check that the
5131 : // arguments are correct. Upon fail it will hit a breakpoint.
5132 :
5133 0 : MIRGraph& mir = gen->graph();
5134 96 : MResumePoint* rp = mir.entryResumePoint();
5135 :
5136 : // No registers are allocated yet, so it's safe to grab anything.
5137 0 : AllocatableGeneralRegisterSet temps(GeneralRegisterSet::All());
5138 0 : Register temp1 = temps.takeAny();
5139 96 : Register temp2 = temps.takeAny();
5140 :
5141 96 : const CompileInfo& info = gen->info();
5142 :
5143 0 : Label miss;
5144 334 : for (uint32_t i = info.startArgSlot(); i < info.endArgSlot(); i++) {
5145 : // All initial parameters are guaranteed to be MParameters.
5146 0 : MParameter* param = rp->getOperand(i)->toParameter();
5147 0 : const TypeSet* types = param->resultTypeSet();
5148 0 : if (!types || types->unknown())
5149 0 : continue;
5150 :
5151 : #ifndef JS_CODEGEN_ARM64
5152 : // Calculate the offset on the stack of the argument.
5153 : // (i - info.startArgSlot()) - Compute index of arg within arg vector.
5154 : // ... * sizeof(Value) - Scale by value size.
5155 : // ArgToStackOffset(...) - Compute displacement within arg vector.
5156 0 : int32_t offset = ArgToStackOffset((i - info.startArgSlot()) * sizeof(Value));
5157 476 : Address argAddr(masm.getStackPointer(), offset);
5158 :
5159 : // guardObjectType will zero the stack pointer register on speculative
5160 : // paths.
5161 238 : Register spectreRegToZero = masm.getStackPointer();
5162 : masm.guardTypeSet(argAddr, types, BarrierKind::TypeSet, temp1, temp2,
5163 238 : spectreRegToZero, &miss);
5164 : #else
5165 : // On ARM64, the stack pointer situation is more complicated. When we
5166 : // enable Ion, we should figure out how to mitigate Spectre there.
5167 : mozilla::Unused << temp1;
5168 : mozilla::Unused << temp2;
5169 : MOZ_CRASH("NYI");
5170 : #endif
5171 : }
5172 :
5173 0 : if (miss.used()) {
5174 96 : if (assert) {
5175 : #ifdef DEBUG
5176 0 : Label success;
5177 0 : masm.jump(&success);
5178 48 : masm.bind(&miss);
5179 :
5180 : // Check for cases where the type set guard might have missed due to
5181 : // changing object groups.
5182 0 : for (uint32_t i = info.startArgSlot(); i < info.endArgSlot(); i++) {
5183 0 : MParameter* param = rp->getOperand(i)->toParameter();
5184 0 : const TemporaryTypeSet* types = param->resultTypeSet();
5185 0 : if (!types || types->unknown())
5186 0 : continue;
5187 :
5188 0 : Label skip;
5189 0 : Address addr(masm.getStackPointer(), ArgToStackOffset((i - info.startArgSlot()) * sizeof(Value)));
5190 0 : masm.branchTestObject(Assembler::NotEqual, addr, &skip);
5191 0 : Register obj = masm.extractObject(addr, temp1);
5192 0 : masm.guardTypeSetMightBeIncomplete(types, obj, temp1, &success);
5193 119 : masm.bind(&skip);
5194 : }
5195 :
5196 0 : masm.assumeUnreachable("Argument check fail.");
5197 48 : masm.bind(&success);
5198 : #else
5199 : MOZ_CRASH("Shouldn't get here in opt builds");
5200 : #endif
5201 : } else {
5202 48 : bailoutFrom(&miss, graph.entrySnapshot());
5203 : }
5204 : }
5205 96 : }
5206 :
5207 : // Out-of-line path to report over-recursed error and fail.
5208 : class CheckOverRecursedFailure : public OutOfLineCodeBase<CodeGenerator>
5209 : {
5210 : LInstruction* lir_;
5211 :
5212 : public:
5213 : explicit CheckOverRecursedFailure(LInstruction* lir)
5214 88 : : lir_(lir)
5215 : { }
5216 :
5217 0 : void accept(CodeGenerator* codegen) override {
5218 0 : codegen->visitCheckOverRecursedFailure(this);
5219 44 : }
5220 :
5221 : LInstruction* lir() const {
5222 : return lir_;
5223 : }
5224 : };
5225 :
5226 : void
5227 48 : CodeGenerator::visitCheckOverRecursed(LCheckOverRecursed* lir)
5228 : {
5229 : // If we don't push anything on the stack, skip the check.
5230 48 : if (omitOverRecursedCheck())
5231 : return;
5232 :
5233 : // Ensure that this frame will not cross the stack limit.
5234 : // This is a weak check, justified by Ion using the C stack: we must always
5235 : // be some distance away from the actual limit, since if the limit is
5236 : // crossed, an error must be thrown, which requires more frames.
5237 : //
5238 : // It must always be possible to trespass past the stack limit.
5239 : // Ion may legally place frames very close to the limit. Calling additional
5240 : // C functions may then violate the limit without any checking.
5241 : //
5242 : // Since Ion frames exist on the C stack, the stack limit may be
5243 : // dynamically set by JS_SetThreadStackLimit() and JS_SetNativeStackQuota().
5244 :
5245 0 : CheckOverRecursedFailure* ool = new(alloc()) CheckOverRecursedFailure(lir);
5246 88 : addOutOfLineCode(ool, lir->mir());
5247 :
5248 : // Conditional forward (unlikely) branch to failure.
5249 0 : const void* limitAddr = gen->runtime->addressOfJitStackLimit();
5250 0 : masm.branchStackPtrRhs(Assembler::AboveOrEqual, AbsoluteAddress(limitAddr), ool->entry());
5251 88 : masm.bind(ool->rejoin());
5252 : }
5253 :
5254 : typedef bool (*DefVarFn)(JSContext*, HandlePropertyName, unsigned, HandleObject);
5255 2 : static const VMFunction DefVarInfo = FunctionInfo<DefVarFn>(DefVar, "DefVar");
5256 :
5257 : void
5258 0 : CodeGenerator::visitDefVar(LDefVar* lir)
5259 : {
5260 0 : Register envChain = ToRegister(lir->environmentChain());
5261 :
5262 0 : pushArg(envChain); // JSObject*
5263 0 : pushArg(Imm32(lir->mir()->attrs())); // unsigned
5264 0 : pushArg(ImmGCPtr(lir->mir()->name())); // PropertyName*
5265 :
5266 0 : callVM(DefVarInfo, lir);
5267 0 : }
5268 :
5269 : typedef bool (*DefLexicalFn)(JSContext*, HandlePropertyName, unsigned);
5270 0 : static const VMFunction DefLexicalInfo =
5271 3 : FunctionInfo<DefLexicalFn>(DefGlobalLexical, "DefGlobalLexical");
5272 :
5273 : void
5274 0 : CodeGenerator::visitDefLexical(LDefLexical* lir)
5275 : {
5276 0 : pushArg(Imm32(lir->mir()->attrs())); // unsigned
5277 0 : pushArg(ImmGCPtr(lir->mir()->name())); // PropertyName*
5278 :
5279 0 : callVM(DefLexicalInfo, lir);
5280 0 : }
5281 :
5282 : typedef bool (*DefFunOperationFn)(JSContext*, HandleScript, HandleObject, HandleFunction);
5283 0 : static const VMFunction DefFunOperationInfo =
5284 3 : FunctionInfo<DefFunOperationFn>(DefFunOperation, "DefFunOperation");
5285 :
5286 : void
5287 0 : CodeGenerator::visitDefFun(LDefFun* lir)
5288 : {
5289 0 : Register envChain = ToRegister(lir->environmentChain());
5290 :
5291 0 : Register fun = ToRegister(lir->fun());
5292 0 : pushArg(fun);
5293 0 : pushArg(envChain);
5294 0 : pushArg(ImmGCPtr(current->mir()->info().script()));
5295 :
5296 0 : callVM(DefFunOperationInfo, lir);
5297 0 : }
5298 :
5299 : typedef bool (*CheckOverRecursedFn)(JSContext*);
5300 0 : static const VMFunction CheckOverRecursedInfo =
5301 3 : FunctionInfo<CheckOverRecursedFn>(CheckOverRecursed, "CheckOverRecursed");
5302 :
5303 : void
5304 44 : CodeGenerator::visitCheckOverRecursedFailure(CheckOverRecursedFailure* ool)
5305 : {
5306 : // The OOL path is hit if the recursion depth has been exceeded.
5307 : // Throw an InternalError for over-recursion.
5308 :
5309 : // LFunctionEnvironment can appear before LCheckOverRecursed, so we have
5310 : // to save all live registers to avoid crashes if CheckOverRecursed triggers
5311 : // a GC.
5312 44 : saveLive(ool->lir());
5313 :
5314 44 : callVM(CheckOverRecursedInfo, ool->lir());
5315 :
5316 0 : restoreLive(ool->lir());
5317 0 : masm.jump(ool->rejoin());
5318 44 : }
5319 :
5320 : IonScriptCounts*
5321 48 : CodeGenerator::maybeCreateScriptCounts()
5322 : {
5323 : // If scripts are being profiled, create a new IonScriptCounts for the
5324 : // profiling data, which will be attached to the associated JSScript or
5325 : // wasm module after code generation finishes.
5326 96 : if (!gen->hasProfilingScripts())
5327 : return nullptr;
5328 :
5329 : // This test inhibits IonScriptCount creation for wasm code which is
5330 : // currently incompatible with wasm codegen for two reasons: (1) wasm code
5331 : // must be serializable and script count codegen bakes in absolute
5332 : // addresses, (2) wasm code does not have a JSScript with which to associate
5333 : // code coverage data.
5334 0 : JSScript* script = gen->info().script();
5335 0 : if (!script)
5336 : return nullptr;
5337 :
5338 0 : UniquePtr<IonScriptCounts> counts(js_new<IonScriptCounts>());
5339 0 : if (!counts || !counts->init(graph.numBlocks()))
5340 : return nullptr;
5341 :
5342 0 : for (size_t i = 0; i < graph.numBlocks(); i++) {
5343 0 : MBasicBlock* block = graph.getBlock(i)->mir();
5344 :
5345 0 : uint32_t offset = 0;
5346 0 : char* description = nullptr;
5347 0 : if (MResumePoint* resume = block->entryResumePoint()) {
5348 : // Find a PC offset in the outermost script to use. If this
5349 : // block is from an inlined script, find a location in the
5350 : // outer script to associate information about the inlining
5351 : // with.
5352 0 : while (resume->caller())
5353 0 : resume = resume->caller();
5354 0 : offset = script->pcToOffset(resume->pc());
5355 :
5356 0 : if (block->entryResumePoint()->caller()) {
5357 : // Get the filename and line number of the inner script.
5358 0 : JSScript* innerScript = block->info().script();
5359 0 : description = (char*) js_calloc(200);
5360 0 : if (description) {
5361 0 : snprintf(description, 200, "%s:%u",
5362 : innerScript->filename(), innerScript->lineno());
5363 : }
5364 : }
5365 : }
5366 :
5367 0 : if (!counts->block(i).init(block->id(), offset, description, block->numSuccessors()))
5368 : return nullptr;
5369 :
5370 0 : for (size_t j = 0; j < block->numSuccessors(); j++)
5371 0 : counts->block(i).setSuccessor(j, skipTrivialBlocks(block->getSuccessor(j))->id());
5372 : }
5373 :
5374 0 : scriptCounts_ = counts.release();
5375 0 : return scriptCounts_;
5376 : }
5377 :
5378 : // Structure for managing the state tracked for a block by script counters.
5379 : struct ScriptCountBlockState
5380 : {
5381 : IonBlockCounts& block;
5382 : MacroAssembler& masm;
5383 :
5384 : Sprinter printer;
5385 :
5386 : public:
5387 : ScriptCountBlockState(IonBlockCounts* block, MacroAssembler* masm)
5388 0 : : block(*block), masm(*masm), printer(GetJitContext()->cx, false)
5389 : {
5390 : }
5391 :
5392 0 : bool init()
5393 : {
5394 0 : if (!printer.init())
5395 : return false;
5396 :
5397 : // Bump the hit count for the block at the start. This code is not
5398 : // included in either the text for the block or the instruction byte
5399 : // counts.
5400 0 : masm.inc64(AbsoluteAddress(block.addressOfHitCount()));
5401 :
5402 : // Collect human readable assembly for the code generated in the block.
5403 0 : masm.setPrinter(&printer);
5404 :
5405 0 : return true;
5406 : }
5407 :
5408 0 : void visitInstruction(LInstruction* ins)
5409 : {
5410 : #ifdef JS_JITSPEW
5411 : // Prefix stream of assembly instructions with their LIR instruction
5412 : // name and any associated high level info.
5413 0 : if (const char* extra = ins->getExtraName())
5414 0 : printer.printf("[%s:%s]\n", ins->opName(), extra);
5415 : else
5416 0 : printer.printf("[%s]\n", ins->opName());
5417 : #endif
5418 0 : }
5419 :
5420 0 : ~ScriptCountBlockState()
5421 0 : {
5422 0 : masm.setPrinter(nullptr);
5423 :
5424 0 : if (!printer.hadOutOfMemory())
5425 0 : block.setCode(printer.string());
5426 0 : }
5427 : };
5428 :
5429 : void
5430 2454 : CodeGenerator::branchIfInvalidated(Register temp, Label* invalidated)
5431 : {
5432 0 : CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), temp);
5433 4908 : masm.propagateOOM(ionScriptLabels_.append(label));
5434 :
5435 : // If IonScript::invalidationCount_ != 0, the script has been invalidated.
5436 0 : masm.branch32(Assembler::NotEqual,
5437 7362 : Address(temp, IonScript::offsetOfInvalidationCount()),
5438 : Imm32(0),
5439 0 : invalidated);
5440 2454 : }
5441 :
5442 : #ifdef DEBUG
5443 : void
5444 1257 : CodeGenerator::emitAssertObjectOrStringResult(Register input, MIRType type, const TemporaryTypeSet* typeset)
5445 : {
5446 1257 : MOZ_ASSERT(type == MIRType::Object || type == MIRType::ObjectOrNull ||
5447 : type == MIRType::String || type == MIRType::Symbol);
5448 :
5449 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
5450 1257 : regs.take(input);
5451 :
5452 0 : Register temp = regs.takeAny();
5453 2514 : masm.push(temp);
5454 :
5455 : // Don't check if the script has been invalidated. In that case invalid
5456 : // types are expected (until we reach the OsiPoint and bailout).
5457 0 : Label done;
5458 1257 : branchIfInvalidated(temp, &done);
5459 :
5460 0 : if ((type == MIRType::Object || type == MIRType::ObjectOrNull) &&
5461 2435 : typeset && !typeset->unknownObject())
5462 : {
5463 : // We have a result TypeSet, assert this object is in it.
5464 0 : Label miss, ok;
5465 0 : if (type == MIRType::ObjectOrNull)
5466 0 : masm.branchPtr(Assembler::Equal, input, ImmWord(0), &ok);
5467 0 : if (typeset->getObjectCount() > 0)
5468 477 : masm.guardObjectType(input, typeset, temp, input, &miss);
5469 : else
5470 0 : masm.jump(&miss);
5471 1098 : masm.jump(&ok);
5472 :
5473 0 : masm.bind(&miss);
5474 549 : masm.guardTypeSetMightBeIncomplete(typeset, input, temp, &ok);
5475 :
5476 549 : masm.assumeUnreachable("MIR instruction returned object with unexpected type");
5477 :
5478 549 : masm.bind(&ok);
5479 : }
5480 :
5481 : // Check that we have a valid GC pointer.
5482 0 : if (JitOptions.fullDebugChecks) {
5483 0 : saveVolatile();
5484 0 : masm.setupUnalignedABICall(temp);
5485 0 : masm.loadJSContext(temp);
5486 0 : masm.passABIArg(temp);
5487 0 : masm.passABIArg(input);
5488 :
5489 : void* callee;
5490 0 : switch (type) {
5491 : case MIRType::Object:
5492 : callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidObjectPtr);
5493 : break;
5494 : case MIRType::ObjectOrNull:
5495 0 : callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidObjectOrNullPtr);
5496 0 : break;
5497 : case MIRType::String:
5498 0 : callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidStringPtr);
5499 0 : break;
5500 : case MIRType::Symbol:
5501 0 : callee = JS_FUNC_TO_DATA_PTR(void*, AssertValidSymbolPtr);
5502 0 : break;
5503 : default:
5504 0 : MOZ_CRASH();
5505 : }
5506 :
5507 0 : masm.callWithABI(callee);
5508 0 : restoreVolatile();
5509 : }
5510 :
5511 0 : masm.bind(&done);
5512 0 : masm.pop(temp);
5513 1257 : }
5514 :
5515 : void
5516 1197 : CodeGenerator::emitAssertResultV(const ValueOperand input, const TemporaryTypeSet* typeset)
5517 : {
5518 0 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
5519 1197 : regs.take(input);
5520 :
5521 0 : Register temp1 = regs.takeAny();
5522 0 : Register temp2 = regs.takeAny();
5523 0 : masm.push(temp1);
5524 2394 : masm.push(temp2);
5525 :
5526 : // Don't check if the script has been invalidated. In that case invalid
5527 : // types are expected (until we reach the OsiPoint and bailout).
5528 0 : Label done;
5529 1197 : branchIfInvalidated(temp1, &done);
5530 :
5531 1658 : if (typeset && !typeset->unknown()) {
5532 : // We have a result TypeSet, assert this value is in it.
5533 0 : Label miss, ok;
5534 0 : masm.guardTypeSet(input, typeset, BarrierKind::TypeSet, temp1, temp2,
5535 0 : input.payloadOrValueReg(), &miss);
5536 852 : masm.jump(&ok);
5537 :
5538 426 : masm.bind(&miss);
5539 :
5540 : // Check for cases where the type set guard might have missed due to
5541 : // changing object groups.
5542 0 : Label realMiss;
5543 0 : masm.branchTestObject(Assembler::NotEqual, input, &realMiss);
5544 0 : Register payload = masm.extractObject(input, temp1);
5545 0 : masm.guardTypeSetMightBeIncomplete(typeset, payload, temp1, &ok);
5546 426 : masm.bind(&realMiss);
5547 :
5548 426 : masm.assumeUnreachable("MIR instruction returned value with unexpected type");
5549 :
5550 426 : masm.bind(&ok);
5551 : }
5552 :
5553 : // Check that we have a valid GC pointer.
5554 0 : if (JitOptions.fullDebugChecks) {
5555 0 : saveVolatile();
5556 :
5557 0 : masm.pushValue(input);
5558 0 : masm.moveStackPtrTo(temp1);
5559 :
5560 0 : masm.setupUnalignedABICall(temp2);
5561 0 : masm.loadJSContext(temp2);
5562 0 : masm.passABIArg(temp2);
5563 0 : masm.passABIArg(temp1);
5564 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, AssertValidValue));
5565 0 : masm.popValue(input);
5566 0 : restoreVolatile();
5567 : }
5568 :
5569 0 : masm.bind(&done);
5570 0 : masm.pop(temp2);
5571 0 : masm.pop(temp1);
5572 1197 : }
5573 :
5574 : void
5575 1266 : CodeGenerator::emitObjectOrStringResultChecks(LInstruction* lir, MDefinition* mir)
5576 : {
5577 0 : if (lir->numDefs() == 0)
5578 9 : return;
5579 :
5580 0 : MOZ_ASSERT(lir->numDefs() == 1);
5581 2514 : if (lir->getDef(0)->isBogusTemp())
5582 : return;
5583 :
5584 0 : Register output = ToRegister(lir->getDef(0));
5585 1257 : emitAssertObjectOrStringResult(output, mir->type(), mir->resultTypeSet());
5586 : }
5587 :
5588 : void
5589 1506 : CodeGenerator::emitValueResultChecks(LInstruction* lir, MDefinition* mir)
5590 : {
5591 0 : if (lir->numDefs() == 0)
5592 309 : return;
5593 :
5594 0 : MOZ_ASSERT(lir->numDefs() == BOX_PIECES);
5595 3008 : if (!lir->getDef(0)->output()->isRegister())
5596 : return;
5597 :
5598 1197 : ValueOperand output = ToOutValue(lir);
5599 :
5600 1197 : emitAssertResultV(output, mir->resultTypeSet());
5601 : }
5602 :
5603 : void
5604 10202 : CodeGenerator::emitDebugResultChecks(LInstruction* ins)
5605 : {
5606 : // In debug builds, check that LIR instructions return valid values.
5607 :
5608 0 : MDefinition* mir = ins->mirRaw();
5609 10202 : if (!mir)
5610 : return;
5611 :
5612 5104 : switch (mir->type()) {
5613 : case MIRType::Object:
5614 : case MIRType::ObjectOrNull:
5615 : case MIRType::String:
5616 : case MIRType::Symbol:
5617 0 : emitObjectOrStringResultChecks(ins, mir);
5618 1266 : break;
5619 : case MIRType::Value:
5620 0 : emitValueResultChecks(ins, mir);
5621 1506 : break;
5622 : default:
5623 : break;
5624 : }
5625 : }
5626 :
5627 : void
5628 10202 : CodeGenerator::emitDebugForceBailing(LInstruction* lir)
5629 : {
5630 0 : if (!lir->snapshot())
5631 0 : return;
5632 4918 : if (lir->isStart())
5633 : return;
5634 4820 : if (lir->isOsiPoint())
5635 : return;
5636 :
5637 0 : masm.comment("emitDebugForceBailing");
5638 3402 : const void* bailAfterAddr = gen->realm->zone()->addressOfIonBailAfter();
5639 :
5640 2268 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
5641 :
5642 0 : Label done, notBail, bail;
5643 1134 : masm.branch32(Assembler::Equal, AbsoluteAddress(bailAfterAddr), Imm32(0), &done);
5644 : {
5645 1134 : Register temp = regs.takeAny();
5646 :
5647 0 : masm.push(temp);
5648 0 : masm.load32(AbsoluteAddress(bailAfterAddr), temp);
5649 0 : masm.sub32(Imm32(1), temp);
5650 1134 : masm.store32(temp, AbsoluteAddress(bailAfterAddr));
5651 :
5652 2268 : masm.branch32(Assembler::NotEqual, temp, Imm32(0), ¬Bail);
5653 : {
5654 0 : masm.pop(temp);
5655 0 : masm.jump(&bail);
5656 1134 : bailoutFrom(&bail, lir->snapshot());
5657 : }
5658 0 : masm.bind(¬Bail);
5659 2268 : masm.pop(temp);
5660 : }
5661 1134 : masm.bind(&done);
5662 : }
5663 : #endif
5664 :
5665 : static void
5666 0 : DumpTrackedSite(const BytecodeSite* site)
5667 : {
5668 0 : if (!JitSpewEnabled(JitSpew_OptimizationTracking))
5669 0 : return;
5670 :
5671 : #ifdef JS_JITSPEW
5672 0 : unsigned column = 0;
5673 0 : unsigned lineNumber = PCToLineNumber(site->script(), site->pc(), &column);
5674 0 : JitSpew(JitSpew_OptimizationTracking, "Types for %s at %s:%u:%u",
5675 0 : CodeName[JSOp(*site->pc())],
5676 : site->script()->filename(),
5677 : lineNumber,
5678 0 : column);
5679 : #endif
5680 : }
5681 :
5682 : static void
5683 0 : DumpTrackedOptimizations(TrackedOptimizations* optimizations)
5684 : {
5685 0 : if (!JitSpewEnabled(JitSpew_OptimizationTracking))
5686 : return;
5687 :
5688 0 : optimizations->spew(JitSpew_OptimizationTracking);
5689 : }
5690 :
5691 : bool
5692 48 : CodeGenerator::generateBody()
5693 : {
5694 48 : IonScriptCounts* counts = maybeCreateScriptCounts();
5695 :
5696 : #if defined(JS_ION_PERF)
5697 : PerfSpewer* perfSpewer = &perfSpewer_;
5698 : if (gen->compilingWasm())
5699 : perfSpewer = &gen->perfSpewer();
5700 : #endif
5701 :
5702 0 : for (size_t i = 0; i < graph.numBlocks(); i++) {
5703 4660 : current = graph.getBlock(i);
5704 :
5705 : // Don't emit any code for trivial blocks, containing just a goto. Such
5706 : // blocks are created to split critical edges, and if we didn't end up
5707 : // putting any instructions in them, we can skip them.
5708 0 : if (current->isTrivial())
5709 716 : continue;
5710 :
5711 : #ifdef JS_JITSPEW
5712 0 : const char* filename = nullptr;
5713 0 : size_t lineNumber = 0;
5714 0 : unsigned columnNumber = 0;
5715 0 : if (current->mir()->info().script()) {
5716 0 : filename = current->mir()->info().script()->filename();
5717 0 : if (current->mir()->pc())
5718 1614 : lineNumber = PCToLineNumber(current->mir()->info().script(), current->mir()->pc(),
5719 : &columnNumber);
5720 : } else {
5721 : #ifdef DEBUG
5722 0 : lineNumber = current->mir()->lineno();
5723 0 : columnNumber = current->mir()->columnIndex();
5724 : #endif
5725 : }
5726 3228 : JitSpew(JitSpew_Codegen, "# block%zu %s:%zu:%u%s:",
5727 : i, filename ? filename : "?", lineNumber, columnNumber,
5728 6456 : current->mir()->isLoopHeader() ? " (loop header)" : "");
5729 : #endif
5730 :
5731 1614 : masm.bind(current->label());
5732 :
5733 0 : mozilla::Maybe<ScriptCountBlockState> blockCounts;
5734 0 : if (counts) {
5735 0 : blockCounts.emplace(&counts->block(i), &masm);
5736 0 : if (!blockCounts->init())
5737 0 : return false;
5738 : }
5739 1614 : TrackedOptimizations* last = nullptr;
5740 :
5741 : #if defined(JS_ION_PERF)
5742 : if (!perfSpewer->startBasicBlock(current->mir(), masm))
5743 : return false;
5744 : #endif
5745 :
5746 0 : for (LInstructionIterator iter = current->begin(); iter != current->end(); iter++) {
5747 20404 : if (!alloc().ensureBallast())
5748 : return false;
5749 :
5750 : #ifdef JS_JITSPEW
5751 0 : JitSpewStart(JitSpew_Codegen, "instruction %s", iter->opName());
5752 0 : if (const char* extra = iter->getExtraName())
5753 0 : JitSpewCont(JitSpew_Codegen, ":%s", extra);
5754 10202 : JitSpewFin(JitSpew_Codegen);
5755 : #endif
5756 :
5757 0 : if (counts)
5758 0 : blockCounts->visitInstruction(*iter);
5759 :
5760 : #ifdef CHECK_OSIPOINT_REGISTERS
5761 0 : if (iter->safepoint())
5762 1276 : resetOsiPointRegs(iter->safepoint());
5763 : #endif
5764 :
5765 10202 : if (iter->mirRaw()) {
5766 : // Only add instructions that have a tracked inline script tree.
5767 0 : if (iter->mirRaw()->trackedTree()) {
5768 5104 : if (!addNativeToBytecodeEntry(iter->mirRaw()->trackedSite()))
5769 : return false;
5770 : }
5771 :
5772 : // Track the start native offset of optimizations.
5773 1 : if (iter->mirRaw()->trackedOptimizations()) {
5774 0 : if (last != iter->mirRaw()->trackedOptimizations()) {
5775 0 : DumpTrackedSite(iter->mirRaw()->trackedSite());
5776 0 : DumpTrackedOptimizations(iter->mirRaw()->trackedOptimizations());
5777 0 : last = iter->mirRaw()->trackedOptimizations();
5778 : }
5779 0 : if (!addTrackedOptimizationsEntry(iter->mirRaw()->trackedOptimizations()))
5780 : return false;
5781 : }
5782 : }
5783 :
5784 10202 : setElement(*iter); // needed to encode correct snapshot location.
5785 :
5786 : #ifdef DEBUG
5787 10202 : emitDebugForceBailing(*iter);
5788 : #endif
5789 :
5790 20404 : switch (iter->op()) {
5791 : #ifndef JS_CODEGEN_NONE
5792 : # define LIROP(op) case LNode::Opcode::op: visit##op(iter->to##op()); break;
5793 135 : LIR_OPCODE_LIST(LIROP)
5794 : # undef LIROP
5795 : #endif
5796 : case LNode::Opcode::Invalid:
5797 : default:
5798 0 : MOZ_CRASH("Invalid LIR op");
5799 : }
5800 :
5801 : // Track the end native offset of optimizations.
5802 1 : if (iter->mirRaw() && iter->mirRaw()->trackedOptimizations())
5803 0 : extendTrackedOptimizationsEntry(iter->mirRaw()->trackedOptimizations());
5804 :
5805 : #ifdef DEBUG
5806 0 : if (!counts)
5807 10202 : emitDebugResultChecks(*iter);
5808 : #endif
5809 : }
5810 3228 : if (masm.oom())
5811 : return false;
5812 :
5813 : #if defined(JS_ION_PERF)
5814 : perfSpewer->endBasicBlock(masm);
5815 : #endif
5816 : }
5817 :
5818 : return true;
5819 : }
5820 :
5821 : // Out-of-line object allocation for LNewArray.
5822 : class OutOfLineNewArray : public OutOfLineCodeBase<CodeGenerator>
5823 : {
5824 : LNewArray* lir_;
5825 :
5826 : public:
5827 : explicit OutOfLineNewArray(LNewArray* lir)
5828 106 : : lir_(lir)
5829 : { }
5830 :
5831 0 : void accept(CodeGenerator* codegen) override {
5832 0 : codegen->visitOutOfLineNewArray(this);
5833 53 : }
5834 :
5835 : LNewArray* lir() const {
5836 : return lir_;
5837 : }
5838 : };
5839 :
5840 : typedef JSObject* (*NewArrayOperationFn)(JSContext*, HandleScript, jsbytecode*, uint32_t,
5841 : NewObjectKind);
5842 0 : static const VMFunction NewArrayOperationInfo =
5843 3 : FunctionInfo<NewArrayOperationFn>(NewArrayOperation, "NewArrayOperation");
5844 :
5845 : static JSObject*
5846 0 : NewArrayWithGroup(JSContext* cx, uint32_t length, HandleObjectGroup group,
5847 : bool convertDoubleElements)
5848 : {
5849 0 : ArrayObject* res = NewFullyAllocatedArrayTryUseGroup(cx, group, length);
5850 0 : if (!res)
5851 : return nullptr;
5852 0 : if (convertDoubleElements)
5853 0 : res->setShouldConvertDoubleElements();
5854 : return res;
5855 : }
5856 :
5857 : typedef JSObject* (*NewArrayWithGroupFn)(JSContext*, uint32_t, HandleObjectGroup, bool);
5858 0 : static const VMFunction NewArrayWithGroupInfo =
5859 3 : FunctionInfo<NewArrayWithGroupFn>(NewArrayWithGroup, "NewArrayWithGroup");
5860 :
5861 : void
5862 53 : CodeGenerator::visitNewArrayCallVM(LNewArray* lir)
5863 : {
5864 106 : Register objReg = ToRegister(lir->output());
5865 :
5866 0 : MOZ_ASSERT(!lir->isCall());
5867 53 : saveLive(lir);
5868 :
5869 106 : JSObject* templateObject = lir->mir()->templateObject();
5870 :
5871 0 : if (templateObject) {
5872 0 : pushArg(Imm32(lir->mir()->convertDoubleElements()));
5873 0 : pushArg(ImmGCPtr(templateObject->group()));
5874 212 : pushArg(Imm32(lir->mir()->length()));
5875 :
5876 53 : callVM(NewArrayWithGroupInfo, lir);
5877 : } else {
5878 0 : pushArg(Imm32(GenericObject));
5879 0 : pushArg(Imm32(lir->mir()->length()));
5880 0 : pushArg(ImmPtr(lir->mir()->pc()));
5881 0 : pushArg(ImmGCPtr(lir->mir()->block()->info().script()));
5882 :
5883 0 : callVM(NewArrayOperationInfo, lir);
5884 : }
5885 :
5886 0 : if (ReturnReg != objReg)
5887 46 : masm.movePtr(ReturnReg, objReg);
5888 :
5889 0 : restoreLive(lir);
5890 53 : }
5891 :
5892 : typedef JSObject* (*NewDerivedTypedObjectFn)(JSContext*,
5893 : HandleObject type,
5894 : HandleObject owner,
5895 : int32_t offset);
5896 0 : static const VMFunction CreateDerivedTypedObjInfo =
5897 3 : FunctionInfo<NewDerivedTypedObjectFn>(CreateDerivedTypedObj, "CreateDerivedTypedObj");
5898 :
5899 : void
5900 0 : CodeGenerator::visitNewDerivedTypedObject(LNewDerivedTypedObject* lir)
5901 : {
5902 0 : pushArg(ToRegister(lir->offset()));
5903 0 : pushArg(ToRegister(lir->owner()));
5904 0 : pushArg(ToRegister(lir->type()));
5905 0 : callVM(CreateDerivedTypedObjInfo, lir);
5906 0 : }
5907 :
5908 : void
5909 0 : CodeGenerator::visitAtan2D(LAtan2D* lir)
5910 : {
5911 0 : Register temp = ToRegister(lir->temp());
5912 0 : FloatRegister y = ToFloatRegister(lir->y());
5913 0 : FloatRegister x = ToFloatRegister(lir->x());
5914 :
5915 0 : masm.setupUnalignedABICall(temp);
5916 0 : masm.passABIArg(y, MoveOp::DOUBLE);
5917 0 : masm.passABIArg(x, MoveOp::DOUBLE);
5918 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaAtan2), MoveOp::DOUBLE);
5919 :
5920 0 : MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg);
5921 0 : }
5922 :
5923 : void
5924 0 : CodeGenerator::visitHypot(LHypot* lir)
5925 : {
5926 0 : Register temp = ToRegister(lir->temp());
5927 0 : uint32_t numArgs = lir->numArgs();
5928 0 : masm.setupUnalignedABICall(temp);
5929 :
5930 0 : for (uint32_t i = 0 ; i < numArgs; ++i)
5931 0 : masm.passABIArg(ToFloatRegister(lir->getOperand(i)), MoveOp::DOUBLE);
5932 :
5933 0 : switch(numArgs) {
5934 : case 2:
5935 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaHypot), MoveOp::DOUBLE);
5936 0 : break;
5937 : case 3:
5938 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, hypot3), MoveOp::DOUBLE);
5939 0 : break;
5940 : case 4:
5941 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, hypot4), MoveOp::DOUBLE);
5942 0 : break;
5943 : default:
5944 0 : MOZ_CRASH("Unexpected number of arguments to hypot function.");
5945 : }
5946 0 : MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg);
5947 0 : }
5948 :
5949 : void
5950 53 : CodeGenerator::visitNewArray(LNewArray* lir)
5951 : {
5952 0 : Register objReg = ToRegister(lir->output());
5953 0 : Register tempReg = ToRegister(lir->temp());
5954 212 : DebugOnly<uint32_t> length = lir->mir()->length();
5955 :
5956 53 : MOZ_ASSERT(length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT);
5957 :
5958 0 : if (lir->mir()->isVMCall()) {
5959 0 : visitNewArrayCallVM(lir);
5960 0 : return;
5961 : }
5962 :
5963 0 : OutOfLineNewArray* ool = new(alloc()) OutOfLineNewArray(lir);
5964 106 : addOutOfLineCode(ool, lir->mir());
5965 :
5966 0 : TemplateObject templateObject(lir->mir()->templateObject());
5967 0 : if (lir->mir()->convertDoubleElements())
5968 0 : templateObject.setConvertDoubleElements();
5969 0 : masm.createGCObject(objReg, tempReg, templateObject, lir->mir()->initialHeap(),
5970 53 : ool->entry());
5971 :
5972 106 : masm.bind(ool->rejoin());
5973 : }
5974 :
5975 : void
5976 0 : CodeGenerator::visitOutOfLineNewArray(OutOfLineNewArray* ool)
5977 : {
5978 0 : visitNewArrayCallVM(ool->lir());
5979 1 : masm.jump(ool->rejoin());
5980 0 : }
5981 :
5982 : void
5983 0 : CodeGenerator::visitNewArrayCopyOnWrite(LNewArrayCopyOnWrite* lir)
5984 : {
5985 0 : Register objReg = ToRegister(lir->output());
5986 0 : Register tempReg = ToRegister(lir->temp());
5987 0 : ArrayObject* templateObject = lir->mir()->templateObject();
5988 0 : gc::InitialHeap initialHeap = lir->mir()->initialHeap();
5989 :
5990 : // If we have a template object, we can inline call object creation.
5991 0 : OutOfLineCode* ool = oolCallVM(NewArrayCopyOnWriteInfo, lir,
5992 0 : ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)),
5993 0 : StoreRegisterTo(objReg));
5994 :
5995 0 : TemplateObject templateObj(templateObject);
5996 0 : templateObj.setDenseElementsAreCopyOnWrite();
5997 0 : masm.createGCObject(objReg, tempReg, templateObj, initialHeap, ool->entry());
5998 :
5999 0 : masm.bind(ool->rejoin());
6000 0 : }
6001 :
6002 : typedef ArrayObject* (*ArrayConstructorOneArgFn)(JSContext*, HandleObjectGroup, int32_t length);
6003 0 : static const VMFunction ArrayConstructorOneArgInfo =
6004 3 : FunctionInfo<ArrayConstructorOneArgFn>(ArrayConstructorOneArg, "ArrayConstructorOneArg");
6005 :
6006 : void
6007 4 : CodeGenerator::visitNewArrayDynamicLength(LNewArrayDynamicLength* lir)
6008 : {
6009 0 : Register lengthReg = ToRegister(lir->length());
6010 0 : Register objReg = ToRegister(lir->output());
6011 8 : Register tempReg = ToRegister(lir->temp());
6012 :
6013 0 : JSObject* templateObject = lir->mir()->templateObject();
6014 8 : gc::InitialHeap initialHeap = lir->mir()->initialHeap();
6015 :
6016 0 : OutOfLineCode* ool = oolCallVM(ArrayConstructorOneArgInfo, lir,
6017 0 : ArgList(ImmGCPtr(templateObject->group()), lengthReg),
6018 12 : StoreRegisterTo(objReg));
6019 :
6020 0 : bool canInline = true;
6021 0 : size_t inlineLength = 0;
6022 0 : if (templateObject->as<ArrayObject>().hasFixedElements()) {
6023 0 : size_t numSlots = gc::GetGCKindSlots(templateObject->asTenured().getAllocKind());
6024 4 : inlineLength = numSlots - ObjectElements::VALUES_PER_HEADER;
6025 : } else {
6026 : canInline = false;
6027 : }
6028 :
6029 4 : if (canInline) {
6030 : // Try to do the allocation inline if the template object is big enough
6031 : // for the length in lengthReg. If the length is bigger we could still
6032 : // use the template object and not allocate the elements, but it's more
6033 : // efficient to do a single big allocation than (repeatedly) reallocating
6034 : // the array later on when filling it.
6035 8 : masm.branch32(Assembler::Above, lengthReg, Imm32(inlineLength), ool->entry());
6036 :
6037 0 : TemplateObject templateObj(templateObject);
6038 4 : masm.createGCObject(objReg, tempReg, templateObj, initialHeap, ool->entry());
6039 :
6040 0 : size_t lengthOffset = NativeObject::offsetOfFixedElements() + ObjectElements::offsetOfLength();
6041 8 : masm.store32(lengthReg, Address(objReg, lengthOffset));
6042 : } else {
6043 0 : masm.jump(ool->entry());
6044 : }
6045 :
6046 0 : masm.bind(ool->rejoin());
6047 4 : }
6048 :
6049 : typedef ArrayIteratorObject* (*NewArrayIteratorObjectFn)(JSContext*, NewObjectKind);
6050 0 : static const VMFunction NewArrayIteratorObjectInfo =
6051 3 : FunctionInfo<NewArrayIteratorObjectFn>(NewArrayIteratorObject, "NewArrayIteratorObject");
6052 :
6053 : typedef StringIteratorObject* (*NewStringIteratorObjectFn)(JSContext*, NewObjectKind);
6054 0 : static const VMFunction NewStringIteratorObjectInfo =
6055 3 : FunctionInfo<NewStringIteratorObjectFn>(NewStringIteratorObject, "NewStringIteratorObject");
6056 :
6057 : void
6058 27 : CodeGenerator::visitNewIterator(LNewIterator* lir)
6059 : {
6060 0 : Register objReg = ToRegister(lir->output());
6061 54 : Register tempReg = ToRegister(lir->temp());
6062 :
6063 : OutOfLineCode* ool;
6064 54 : switch (lir->mir()->type()) {
6065 : case MNewIterator::ArrayIterator:
6066 0 : ool = oolCallVM(NewArrayIteratorObjectInfo, lir,
6067 0 : ArgList(Imm32(GenericObject)),
6068 0 : StoreRegisterTo(objReg));
6069 27 : break;
6070 : case MNewIterator::StringIterator:
6071 0 : ool = oolCallVM(NewStringIteratorObjectInfo, lir,
6072 0 : ArgList(Imm32(GenericObject)),
6073 0 : StoreRegisterTo(objReg));
6074 0 : break;
6075 : default:
6076 0 : MOZ_CRASH("unexpected iterator type");
6077 : }
6078 :
6079 0 : TemplateObject templateObject(lir->mir()->templateObject());
6080 27 : masm.createGCObject(objReg, tempReg, templateObject, gc::DefaultHeap, ool->entry());
6081 :
6082 0 : masm.bind(ool->rejoin());
6083 27 : }
6084 :
6085 : typedef TypedArrayObject* (*TypedArrayConstructorOneArgFn)(JSContext*, HandleObject, int32_t length);
6086 0 : static const VMFunction TypedArrayConstructorOneArgInfo =
6087 0 : FunctionInfo<TypedArrayConstructorOneArgFn>(TypedArrayCreateWithTemplate,
6088 1 : "TypedArrayCreateWithTemplate");
6089 :
6090 : void
6091 0 : CodeGenerator::visitNewTypedArray(LNewTypedArray* lir)
6092 : {
6093 0 : Register objReg = ToRegister(lir->output());
6094 0 : Register tempReg = ToRegister(lir->temp1());
6095 0 : Register lengthReg = ToRegister(lir->temp2());
6096 0 : LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
6097 :
6098 0 : JSObject* templateObject = lir->mir()->templateObject();
6099 0 : gc::InitialHeap initialHeap = lir->mir()->initialHeap();
6100 :
6101 0 : TypedArrayObject* ttemplate = &templateObject->as<TypedArrayObject>();
6102 0 : uint32_t n = ttemplate->length();
6103 :
6104 0 : OutOfLineCode* ool = oolCallVM(TypedArrayConstructorOneArgInfo, lir,
6105 0 : ArgList(ImmGCPtr(templateObject), Imm32(n)),
6106 0 : StoreRegisterTo(objReg));
6107 :
6108 0 : TemplateObject templateObj(templateObject);
6109 0 : masm.createGCObject(objReg, tempReg, templateObj, initialHeap, ool->entry());
6110 :
6111 0 : masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(),
6112 0 : ttemplate, MacroAssembler::TypedArrayLength::Fixed);
6113 :
6114 0 : masm.bind(ool->rejoin());
6115 0 : }
6116 :
6117 : void
6118 0 : CodeGenerator::visitNewTypedArrayDynamicLength(LNewTypedArrayDynamicLength* lir)
6119 : {
6120 0 : Register lengthReg = ToRegister(lir->length());
6121 0 : Register objReg = ToRegister(lir->output());
6122 0 : Register tempReg = ToRegister(lir->temp());
6123 0 : LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
6124 :
6125 0 : JSObject* templateObject = lir->mir()->templateObject();
6126 0 : gc::InitialHeap initialHeap = lir->mir()->initialHeap();
6127 :
6128 0 : TypedArrayObject* ttemplate = &templateObject->as<TypedArrayObject>();
6129 :
6130 0 : OutOfLineCode* ool = oolCallVM(TypedArrayConstructorOneArgInfo, lir,
6131 0 : ArgList(ImmGCPtr(templateObject), lengthReg),
6132 0 : StoreRegisterTo(objReg));
6133 :
6134 0 : TemplateObject templateObj(templateObject);
6135 0 : masm.createGCObject(objReg, tempReg, templateObj, initialHeap, ool->entry());
6136 :
6137 0 : masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(),
6138 0 : ttemplate, MacroAssembler::TypedArrayLength::Dynamic);
6139 :
6140 0 : masm.bind(ool->rejoin());
6141 0 : }
6142 :
6143 : // Out-of-line object allocation for JSOP_NEWOBJECT.
6144 : class OutOfLineNewObject : public OutOfLineCodeBase<CodeGenerator>
6145 : {
6146 : LNewObject* lir_;
6147 :
6148 : public:
6149 : explicit OutOfLineNewObject(LNewObject* lir)
6150 74 : : lir_(lir)
6151 : { }
6152 :
6153 0 : void accept(CodeGenerator* codegen) override {
6154 0 : codegen->visitOutOfLineNewObject(this);
6155 37 : }
6156 :
6157 : LNewObject* lir() const {
6158 : return lir_;
6159 : }
6160 : };
6161 :
6162 : typedef JSObject* (*NewInitObjectWithTemplateFn)(JSContext*, HandleObject);
6163 0 : static const VMFunction NewInitObjectWithTemplateInfo =
6164 0 : FunctionInfo<NewInitObjectWithTemplateFn>(NewObjectOperationWithTemplate,
6165 1 : "NewObjectOperationWithTemplate");
6166 :
6167 : typedef JSObject* (*NewInitObjectFn)(JSContext*, HandleScript, jsbytecode* pc, NewObjectKind);
6168 0 : static const VMFunction NewInitObjectInfo =
6169 3 : FunctionInfo<NewInitObjectFn>(NewObjectOperation, "NewObjectOperation");
6170 :
6171 : typedef PlainObject* (*ObjectCreateWithTemplateFn)(JSContext*, HandlePlainObject);
6172 0 : static const VMFunction ObjectCreateWithTemplateInfo =
6173 3 : FunctionInfo<ObjectCreateWithTemplateFn>(ObjectCreateWithTemplate, "ObjectCreateWithTemplate");
6174 :
6175 : void
6176 37 : CodeGenerator::visitNewObjectVMCall(LNewObject* lir)
6177 : {
6178 74 : Register objReg = ToRegister(lir->output());
6179 :
6180 0 : MOZ_ASSERT(!lir->isCall());
6181 37 : saveLive(lir);
6182 :
6183 74 : JSObject* templateObject = lir->mir()->templateObject();
6184 :
6185 : // If we're making a new object with a class prototype (that is, an object
6186 : // that derives its class from its prototype instead of being
6187 : // PlainObject::class_'d) from self-hosted code, we need a different init
6188 : // function.
6189 74 : switch (lir->mir()->mode()) {
6190 : case MNewObject::ObjectLiteral:
6191 0 : if (templateObject) {
6192 0 : pushArg(ImmGCPtr(templateObject));
6193 37 : callVM(NewInitObjectWithTemplateInfo, lir);
6194 : } else {
6195 0 : pushArg(Imm32(GenericObject));
6196 0 : pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
6197 0 : pushArg(ImmGCPtr(lir->mir()->block()->info().script()));
6198 0 : callVM(NewInitObjectInfo, lir);
6199 : }
6200 : break;
6201 : case MNewObject::ObjectCreate:
6202 0 : pushArg(ImmGCPtr(templateObject));
6203 0 : callVM(ObjectCreateWithTemplateInfo, lir);
6204 0 : break;
6205 : }
6206 :
6207 0 : if (ReturnReg != objReg)
6208 33 : masm.movePtr(ReturnReg, objReg);
6209 :
6210 0 : restoreLive(lir);
6211 37 : }
6212 :
6213 : static bool
6214 40 : ShouldInitFixedSlots(LInstruction* lir, const TemplateObject& obj)
6215 : {
6216 80 : if (!obj.isNative())
6217 : return true;
6218 37 : const NativeTemplateObject& templateObj = obj.asNativeTemplateObject();
6219 :
6220 : // Look for StoreFixedSlot instructions following an object allocation
6221 : // that write to this object before a GC is triggered or this object is
6222 : // passed to a VM call. If all fixed slots will be initialized, the
6223 : // allocation code doesn't need to set the slots to |undefined|.
6224 :
6225 0 : uint32_t nfixed = templateObj.numUsedFixedSlots();
6226 37 : if (nfixed == 0)
6227 : return false;
6228 :
6229 : // Only optimize if all fixed slots are initially |undefined|, so that we
6230 : // can assume incremental pre-barriers are not necessary. See also the
6231 : // comment below.
6232 0 : for (uint32_t slot = 0; slot < nfixed; slot++) {
6233 128 : if (!templateObj.getSlot(slot).isUndefined())
6234 : return true;
6235 : }
6236 :
6237 : // Keep track of the fixed slots that are initialized. initializedSlots is
6238 : // a bit mask with a bit for each slot.
6239 28 : MOZ_ASSERT(nfixed <= NativeObject::MAX_FIXED_SLOTS);
6240 : static_assert(NativeObject::MAX_FIXED_SLOTS <= 32, "Slot bits must fit in 32 bits");
6241 0 : uint32_t initializedSlots = 0;
6242 28 : uint32_t numInitialized = 0;
6243 :
6244 0 : MInstruction* allocMir = lir->mirRaw()->toInstruction();
6245 56 : MBasicBlock* block = allocMir->block();
6246 :
6247 : // Skip the allocation instruction.
6248 0 : MInstructionIterator iter = block->begin(allocMir);
6249 0 : MOZ_ASSERT(*iter == allocMir);
6250 28 : iter++;
6251 :
6252 : while (true) {
6253 0 : for (; iter != block->end(); iter++) {
6254 212 : if (iter->isNop() || iter->isConstant() || iter->isPostWriteBarrier()) {
6255 : // These instructions won't trigger a GC or read object slots.
6256 : continue;
6257 : }
6258 :
6259 0 : if (iter->isStoreFixedSlot()) {
6260 0 : MStoreFixedSlot* store = iter->toStoreFixedSlot();
6261 46 : if (store->object() != allocMir)
6262 : return true;
6263 :
6264 : // We may not initialize this object slot on allocation, so the
6265 : // pre-barrier could read uninitialized memory. Simply disable
6266 : // the barrier for this store: the object was just initialized
6267 : // so the barrier is not necessary.
6268 46 : store->setNeedsBarrier(false);
6269 :
6270 0 : uint32_t slot = store->slot();
6271 0 : MOZ_ASSERT(slot < nfixed);
6272 0 : if ((initializedSlots & (1 << slot)) == 0) {
6273 0 : numInitialized++;
6274 46 : initializedSlots |= (1 << slot);
6275 :
6276 46 : if (numInitialized == nfixed) {
6277 : // All fixed slots will be initialized.
6278 23 : MOZ_ASSERT(mozilla::CountPopulation32(initializedSlots) == nfixed);
6279 : return false;
6280 : }
6281 : }
6282 : continue;
6283 : }
6284 :
6285 0 : if (iter->isGoto()) {
6286 0 : block = iter->toGoto()->target();
6287 2 : if (block->numPredecessors() != 1)
6288 : return true;
6289 : break;
6290 : }
6291 :
6292 : // Unhandled instruction, assume it bails or reads object slots.
6293 : return true;
6294 : }
6295 0 : iter = block->begin();
6296 2 : }
6297 :
6298 : MOZ_CRASH("Shouldn't get here");
6299 : }
6300 :
6301 : void
6302 37 : CodeGenerator::visitNewObject(LNewObject* lir)
6303 : {
6304 0 : Register objReg = ToRegister(lir->output());
6305 74 : Register tempReg = ToRegister(lir->temp());
6306 :
6307 1 : if (lir->mir()->isVMCall()) {
6308 0 : visitNewObjectVMCall(lir);
6309 0 : return;
6310 : }
6311 :
6312 0 : OutOfLineNewObject* ool = new(alloc()) OutOfLineNewObject(lir);
6313 74 : addOutOfLineCode(ool, lir->mir());
6314 :
6315 111 : TemplateObject templateObject(lir->mir()->templateObject());
6316 :
6317 0 : bool initContents = ShouldInitFixedSlots(lir, templateObject);
6318 0 : masm.createGCObject(objReg, tempReg, templateObject, lir->mir()->initialHeap(), ool->entry(),
6319 37 : initContents);
6320 :
6321 74 : masm.bind(ool->rejoin());
6322 : }
6323 :
6324 : void
6325 0 : CodeGenerator::visitOutOfLineNewObject(OutOfLineNewObject* ool)
6326 : {
6327 0 : visitNewObjectVMCall(ool->lir());
6328 1 : masm.jump(ool->rejoin());
6329 0 : }
6330 :
6331 : typedef InlineTypedObject* (*NewTypedObjectFn)(JSContext*, Handle<InlineTypedObject*>, gc::InitialHeap);
6332 0 : static const VMFunction NewTypedObjectInfo =
6333 3 : FunctionInfo<NewTypedObjectFn>(InlineTypedObject::createCopy, "InlineTypedObject::createCopy");
6334 :
6335 : void
6336 0 : CodeGenerator::visitNewTypedObject(LNewTypedObject* lir)
6337 : {
6338 0 : Register object = ToRegister(lir->output());
6339 0 : Register temp = ToRegister(lir->temp());
6340 0 : InlineTypedObject* templateObject = lir->mir()->templateObject();
6341 0 : gc::InitialHeap initialHeap = lir->mir()->initialHeap();
6342 :
6343 0 : OutOfLineCode* ool = oolCallVM(NewTypedObjectInfo, lir,
6344 0 : ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)),
6345 0 : StoreRegisterTo(object));
6346 :
6347 0 : TemplateObject templateObj(templateObject);
6348 0 : masm.createGCObject(object, temp, templateObj, initialHeap, ool->entry());
6349 :
6350 0 : masm.bind(ool->rejoin());
6351 0 : }
6352 :
6353 : void
6354 0 : CodeGenerator::visitSimdBox(LSimdBox* lir)
6355 : {
6356 0 : FloatRegister in = ToFloatRegister(lir->input());
6357 0 : Register object = ToRegister(lir->output());
6358 0 : Register temp = ToRegister(lir->temp());
6359 0 : InlineTypedObject* templateObject = lir->mir()->templateObject();
6360 0 : gc::InitialHeap initialHeap = lir->mir()->initialHeap();
6361 0 : MIRType type = lir->mir()->input()->type();
6362 :
6363 0 : addSimdTemplateToReadBarrier(lir->mir()->simdType());
6364 :
6365 0 : MOZ_ASSERT(lir->safepoint()->liveRegs().has(in), "Save the input register across oolCallVM");
6366 0 : OutOfLineCode* ool = oolCallVM(NewTypedObjectInfo, lir,
6367 0 : ArgList(ImmGCPtr(templateObject), Imm32(initialHeap)),
6368 0 : StoreRegisterTo(object));
6369 :
6370 0 : TemplateObject templateObj(templateObject);
6371 0 : masm.createGCObject(object, temp, templateObj, initialHeap, ool->entry());
6372 0 : masm.bind(ool->rejoin());
6373 :
6374 0 : Address objectData(object, InlineTypedObject::offsetOfDataStart());
6375 0 : switch (type) {
6376 : case MIRType::Int8x16:
6377 : case MIRType::Int16x8:
6378 : case MIRType::Int32x4:
6379 : case MIRType::Bool8x16:
6380 : case MIRType::Bool16x8:
6381 : case MIRType::Bool32x4:
6382 0 : masm.storeUnalignedSimd128Int(in, objectData);
6383 0 : break;
6384 : case MIRType::Float32x4:
6385 0 : masm.storeUnalignedSimd128Float(in, objectData);
6386 0 : break;
6387 : default:
6388 0 : MOZ_CRASH("Unknown SIMD kind when generating code for SimdBox.");
6389 : }
6390 0 : }
6391 :
6392 : void
6393 0 : CodeGenerator::addSimdTemplateToReadBarrier(SimdType simdType)
6394 : {
6395 0 : simdTemplatesToReadBarrier_ |= 1 << uint32_t(simdType);
6396 0 : }
6397 :
6398 : void
6399 0 : CodeGenerator::visitSimdUnbox(LSimdUnbox* lir)
6400 : {
6401 0 : Register object = ToRegister(lir->input());
6402 0 : FloatRegister simd = ToFloatRegister(lir->output());
6403 0 : Register temp = ToRegister(lir->temp());
6404 0 : Label bail;
6405 :
6406 0 : masm.branchIfNotSimdObject(object, temp, lir->mir()->simdType(), &bail);
6407 :
6408 : // Load the value from the data of the InlineTypedObject.
6409 0 : Address objectData(object, InlineTypedObject::offsetOfDataStart());
6410 0 : switch (lir->mir()->type()) {
6411 : case MIRType::Int8x16:
6412 : case MIRType::Int16x8:
6413 : case MIRType::Int32x4:
6414 : case MIRType::Bool8x16:
6415 : case MIRType::Bool16x8:
6416 : case MIRType::Bool32x4:
6417 0 : masm.loadUnalignedSimd128Int(objectData, simd);
6418 0 : break;
6419 : case MIRType::Float32x4:
6420 0 : masm.loadUnalignedSimd128Float(objectData, simd);
6421 0 : break;
6422 : default:
6423 0 : MOZ_CRASH("The impossible happened!");
6424 : }
6425 :
6426 0 : bailoutFrom(&bail, lir->snapshot());
6427 0 : }
6428 :
6429 : typedef js::NamedLambdaObject* (*NewNamedLambdaObjectFn)(JSContext*, HandleFunction, gc::InitialHeap);
6430 0 : static const VMFunction NewNamedLambdaObjectInfo =
6431 0 : FunctionInfo<NewNamedLambdaObjectFn>(NamedLambdaObject::createTemplateObject,
6432 1 : "NamedLambdaObject::createTemplateObject");
6433 :
6434 : void
6435 0 : CodeGenerator::visitNewNamedLambdaObject(LNewNamedLambdaObject* lir)
6436 : {
6437 0 : Register objReg = ToRegister(lir->output());
6438 0 : Register tempReg = ToRegister(lir->temp());
6439 0 : const CompileInfo& info = lir->mir()->block()->info();
6440 :
6441 : // If we have a template object, we can inline call object creation.
6442 0 : OutOfLineCode* ool = oolCallVM(NewNamedLambdaObjectInfo, lir,
6443 0 : ArgList(ImmGCPtr(info.funMaybeLazy()), Imm32(gc::DefaultHeap)),
6444 0 : StoreRegisterTo(objReg));
6445 :
6446 0 : TemplateObject templateObject(lir->mir()->templateObj());
6447 :
6448 0 : bool initContents = ShouldInitFixedSlots(lir, templateObject);
6449 0 : masm.createGCObject(objReg, tempReg, templateObject, gc::DefaultHeap, ool->entry(),
6450 0 : initContents);
6451 :
6452 0 : masm.bind(ool->rejoin());
6453 0 : }
6454 :
6455 : typedef JSObject* (*NewCallObjectFn)(JSContext*, HandleShape, HandleObjectGroup);
6456 0 : static const VMFunction NewCallObjectInfo =
6457 3 : FunctionInfo<NewCallObjectFn>(NewCallObject, "NewCallObject");
6458 :
6459 : void
6460 1 : CodeGenerator::visitNewCallObject(LNewCallObject* lir)
6461 : {
6462 0 : Register objReg = ToRegister(lir->output());
6463 2 : Register tempReg = ToRegister(lir->temp());
6464 :
6465 2 : CallObject* templateObj = lir->mir()->templateObject();
6466 :
6467 0 : OutOfLineCode* ool = oolCallVM(NewCallObjectInfo, lir,
6468 0 : ArgList(ImmGCPtr(templateObj->lastProperty()),
6469 0 : ImmGCPtr(templateObj->group())),
6470 3 : StoreRegisterTo(objReg));
6471 :
6472 : // Inline call object creation, using the OOL path only for tricky cases.
6473 0 : TemplateObject templateObject(templateObj);
6474 0 : bool initContents = ShouldInitFixedSlots(lir, templateObject);
6475 0 : masm.createGCObject(objReg, tempReg, templateObject, gc::DefaultHeap, ool->entry(),
6476 1 : initContents);
6477 :
6478 0 : masm.bind(ool->rejoin());
6479 1 : }
6480 :
6481 : typedef JSObject* (*NewSingletonCallObjectFn)(JSContext*, HandleShape);
6482 0 : static const VMFunction NewSingletonCallObjectInfo =
6483 3 : FunctionInfo<NewSingletonCallObjectFn>(NewSingletonCallObject, "NewSingletonCallObject");
6484 :
6485 : void
6486 0 : CodeGenerator::visitNewSingletonCallObject(LNewSingletonCallObject* lir)
6487 : {
6488 0 : Register objReg = ToRegister(lir->output());
6489 :
6490 0 : JSObject* templateObj = lir->mir()->templateObject();
6491 :
6492 : OutOfLineCode* ool;
6493 0 : ool = oolCallVM(NewSingletonCallObjectInfo, lir,
6494 0 : ArgList(ImmGCPtr(templateObj->as<CallObject>().lastProperty())),
6495 0 : StoreRegisterTo(objReg));
6496 :
6497 : // Objects can only be given singleton types in VM calls. We make the call
6498 : // out of line to not bloat inline code, even if (naively) this seems like
6499 : // extra work.
6500 0 : masm.jump(ool->entry());
6501 0 : masm.bind(ool->rejoin());
6502 0 : }
6503 :
6504 : typedef JSObject* (*NewStringObjectFn)(JSContext*, HandleString);
6505 0 : static const VMFunction NewStringObjectInfo =
6506 3 : FunctionInfo<NewStringObjectFn>(NewStringObject, "NewStringObject");
6507 :
6508 : void
6509 0 : CodeGenerator::visitNewStringObject(LNewStringObject* lir)
6510 : {
6511 0 : Register input = ToRegister(lir->input());
6512 0 : Register output = ToRegister(lir->output());
6513 0 : Register temp = ToRegister(lir->temp());
6514 :
6515 0 : StringObject* templateObj = lir->mir()->templateObj();
6516 :
6517 0 : OutOfLineCode* ool = oolCallVM(NewStringObjectInfo, lir, ArgList(input),
6518 0 : StoreRegisterTo(output));
6519 :
6520 0 : TemplateObject templateObject(templateObj);
6521 0 : masm.createGCObject(output, temp, templateObject, gc::DefaultHeap, ool->entry());
6522 :
6523 0 : masm.loadStringLength(input, temp);
6524 :
6525 0 : masm.storeValue(JSVAL_TYPE_STRING, input, Address(output, StringObject::offsetOfPrimitiveValue()));
6526 0 : masm.storeValue(JSVAL_TYPE_INT32, temp, Address(output, StringObject::offsetOfLength()));
6527 :
6528 0 : masm.bind(ool->rejoin());
6529 0 : }
6530 :
6531 : typedef bool(*InitElemFn)(JSContext* cx, jsbytecode* pc, HandleObject obj,
6532 : HandleValue id, HandleValue value);
6533 0 : static const VMFunction InitElemInfo =
6534 3 : FunctionInfo<InitElemFn>(InitElemOperation, "InitElemOperation");
6535 :
6536 : void
6537 0 : CodeGenerator::visitInitElem(LInitElem* lir)
6538 : {
6539 0 : Register objReg = ToRegister(lir->getObject());
6540 :
6541 0 : pushArg(ToValue(lir, LInitElem::ValueIndex));
6542 0 : pushArg(ToValue(lir, LInitElem::IdIndex));
6543 0 : pushArg(objReg);
6544 0 : pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
6545 :
6546 0 : callVM(InitElemInfo, lir);
6547 0 : }
6548 :
6549 : typedef bool (*InitElemGetterSetterFn)(JSContext*, jsbytecode*, HandleObject, HandleValue,
6550 : HandleObject);
6551 0 : static const VMFunction InitElemGetterSetterInfo =
6552 3 : FunctionInfo<InitElemGetterSetterFn>(InitGetterSetterOperation, "InitElemGetterSetterOperation");
6553 :
6554 : void
6555 0 : CodeGenerator::visitInitElemGetterSetter(LInitElemGetterSetter* lir)
6556 : {
6557 0 : Register obj = ToRegister(lir->object());
6558 0 : Register value = ToRegister(lir->value());
6559 :
6560 0 : pushArg(value);
6561 0 : pushArg(ToValue(lir, LInitElemGetterSetter::IdIndex));
6562 0 : pushArg(obj);
6563 0 : pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
6564 :
6565 0 : callVM(InitElemGetterSetterInfo, lir);
6566 0 : }
6567 :
6568 : typedef bool(*MutatePrototypeFn)(JSContext* cx, HandlePlainObject obj, HandleValue value);
6569 0 : static const VMFunction MutatePrototypeInfo =
6570 3 : FunctionInfo<MutatePrototypeFn>(MutatePrototype, "MutatePrototype");
6571 :
6572 : void
6573 0 : CodeGenerator::visitMutateProto(LMutateProto* lir)
6574 : {
6575 0 : Register objReg = ToRegister(lir->getObject());
6576 :
6577 0 : pushArg(ToValue(lir, LMutateProto::ValueIndex));
6578 0 : pushArg(objReg);
6579 :
6580 0 : callVM(MutatePrototypeInfo, lir);
6581 0 : }
6582 :
6583 : typedef bool(*InitPropGetterSetterFn)(JSContext*, jsbytecode*, HandleObject, HandlePropertyName,
6584 : HandleObject);
6585 0 : static const VMFunction InitPropGetterSetterInfo =
6586 3 : FunctionInfo<InitPropGetterSetterFn>(InitGetterSetterOperation, "InitPropGetterSetterOperation");
6587 :
6588 : void
6589 0 : CodeGenerator::visitInitPropGetterSetter(LInitPropGetterSetter* lir)
6590 : {
6591 0 : Register obj = ToRegister(lir->object());
6592 0 : Register value = ToRegister(lir->value());
6593 :
6594 0 : pushArg(value);
6595 0 : pushArg(ImmGCPtr(lir->mir()->name()));
6596 0 : pushArg(obj);
6597 0 : pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
6598 :
6599 0 : callVM(InitPropGetterSetterInfo, lir);
6600 0 : }
6601 :
6602 : typedef bool (*CreateThisFn)(JSContext* cx, HandleObject callee, HandleObject newTarget, MutableHandleValue rval);
6603 2 : static const VMFunction CreateThisInfoCodeGen = FunctionInfo<CreateThisFn>(CreateThis, "CreateThis");
6604 :
6605 : void
6606 2 : CodeGenerator::visitCreateThis(LCreateThis* lir)
6607 : {
6608 0 : const LAllocation* callee = lir->getCallee();
6609 2 : const LAllocation* newTarget = lir->getNewTarget();
6610 :
6611 0 : if (newTarget->isConstant())
6612 0 : pushArg(ImmGCPtr(&newTarget->toConstant()->toObject()));
6613 : else
6614 4 : pushArg(ToRegister(newTarget));
6615 :
6616 0 : if (callee->isConstant())
6617 0 : pushArg(ImmGCPtr(&callee->toConstant()->toObject()));
6618 : else
6619 4 : pushArg(ToRegister(callee));
6620 :
6621 0 : callVM(CreateThisInfoCodeGen, lir);
6622 2 : }
6623 :
6624 : static JSObject*
6625 0 : CreateThisForFunctionWithProtoWrapper(JSContext* cx, HandleObject callee, HandleObject newTarget,
6626 : HandleObject proto)
6627 : {
6628 0 : return CreateThisForFunctionWithProto(cx, callee, newTarget, proto);
6629 : }
6630 :
6631 : typedef JSObject* (*CreateThisWithProtoFn)(JSContext* cx, HandleObject callee,
6632 : HandleObject newTarget, HandleObject proto);
6633 0 : static const VMFunction CreateThisWithProtoInfo =
6634 0 : FunctionInfo<CreateThisWithProtoFn>(CreateThisForFunctionWithProtoWrapper,
6635 1 : "CreateThisForFunctionWithProtoWrapper");
6636 :
6637 : void
6638 0 : CodeGenerator::visitCreateThisWithProto(LCreateThisWithProto* lir)
6639 : {
6640 0 : const LAllocation* callee = lir->getCallee();
6641 0 : const LAllocation* newTarget = lir->getNewTarget();
6642 0 : const LAllocation* proto = lir->getPrototype();
6643 :
6644 0 : if (proto->isConstant())
6645 0 : pushArg(ImmGCPtr(&proto->toConstant()->toObject()));
6646 : else
6647 0 : pushArg(ToRegister(proto));
6648 :
6649 0 : if (newTarget->isConstant())
6650 0 : pushArg(ImmGCPtr(&newTarget->toConstant()->toObject()));
6651 : else
6652 0 : pushArg(ToRegister(newTarget));
6653 :
6654 0 : if (callee->isConstant())
6655 0 : pushArg(ImmGCPtr(&callee->toConstant()->toObject()));
6656 : else
6657 0 : pushArg(ToRegister(callee));
6658 :
6659 0 : callVM(CreateThisWithProtoInfo, lir);
6660 0 : }
6661 :
6662 : void
6663 2 : CodeGenerator::visitCreateThisWithTemplate(LCreateThisWithTemplate* lir)
6664 : {
6665 0 : JSObject* templateObject = lir->mir()->templateObject();
6666 0 : Register objReg = ToRegister(lir->output());
6667 4 : Register tempReg = ToRegister(lir->temp());
6668 :
6669 0 : OutOfLineCode* ool = oolCallVM(NewInitObjectWithTemplateInfo, lir,
6670 0 : ArgList(ImmGCPtr(templateObject)),
6671 6 : StoreRegisterTo(objReg));
6672 :
6673 : // Allocate. If the FreeList is empty, call to VM, which may GC.
6674 0 : TemplateObject templateObj(templateObject);
6675 0 : bool initContents = !templateObj.isPlainObject() || ShouldInitFixedSlots(lir, templateObj);
6676 0 : masm.createGCObject(objReg, tempReg, templateObj, lir->mir()->initialHeap(), ool->entry(),
6677 2 : initContents);
6678 :
6679 0 : masm.bind(ool->rejoin());
6680 2 : }
6681 :
6682 : typedef JSObject* (*NewIonArgumentsObjectFn)(JSContext* cx, JitFrameLayout* frame, HandleObject);
6683 0 : static const VMFunction NewIonArgumentsObjectInfo =
6684 0 : FunctionInfo<NewIonArgumentsObjectFn>((NewIonArgumentsObjectFn) ArgumentsObject::createForIon,
6685 1 : "ArgumentsObject::createForIon");
6686 :
6687 : void
6688 0 : CodeGenerator::visitCreateArgumentsObject(LCreateArgumentsObject* lir)
6689 : {
6690 : // This should be getting constructed in the first block only, and not any OSR entry blocks.
6691 0 : MOZ_ASSERT(lir->mir()->block()->id() == 0);
6692 :
6693 0 : Register callObj = ToRegister(lir->getCallObject());
6694 0 : Register temp = ToRegister(lir->temp0());
6695 0 : Label done;
6696 :
6697 0 : if (ArgumentsObject* templateObj = lir->mir()->templateObject()) {
6698 0 : Register objTemp = ToRegister(lir->temp1());
6699 0 : Register cxTemp = ToRegister(lir->temp2());
6700 :
6701 0 : masm.Push(callObj);
6702 :
6703 : // Try to allocate an arguments object. This will leave the reserved
6704 : // slots uninitialized, so it's important we don't GC until we
6705 : // initialize these slots in ArgumentsObject::finishForIon.
6706 0 : Label failure;
6707 0 : TemplateObject templateObject(templateObj);
6708 0 : masm.createGCObject(objTemp, temp, templateObject, gc::DefaultHeap, &failure,
6709 0 : /* initContents = */ false);
6710 :
6711 0 : masm.moveStackPtrTo(temp);
6712 0 : masm.addPtr(Imm32(masm.framePushed()), temp);
6713 :
6714 0 : masm.setupUnalignedABICall(cxTemp);
6715 0 : masm.loadJSContext(cxTemp);
6716 0 : masm.passABIArg(cxTemp);
6717 0 : masm.passABIArg(temp);
6718 0 : masm.passABIArg(callObj);
6719 0 : masm.passABIArg(objTemp);
6720 :
6721 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ArgumentsObject::finishForIon));
6722 0 : masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &failure);
6723 :
6724 : // Discard saved callObj on the stack.
6725 0 : masm.addToStackPtr(Imm32(sizeof(uintptr_t)));
6726 0 : masm.jump(&done);
6727 :
6728 0 : masm.bind(&failure);
6729 0 : masm.Pop(callObj);
6730 : }
6731 :
6732 0 : masm.moveStackPtrTo(temp);
6733 0 : masm.addPtr(Imm32(frameSize()), temp);
6734 :
6735 0 : pushArg(callObj);
6736 0 : pushArg(temp);
6737 0 : callVM(NewIonArgumentsObjectInfo, lir);
6738 :
6739 0 : masm.bind(&done);
6740 0 : }
6741 :
6742 : void
6743 0 : CodeGenerator::visitGetArgumentsObjectArg(LGetArgumentsObjectArg* lir)
6744 : {
6745 0 : Register temp = ToRegister(lir->getTemp(0));
6746 0 : Register argsObj = ToRegister(lir->getArgsObject());
6747 0 : ValueOperand out = ToOutValue(lir);
6748 :
6749 0 : masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()), temp);
6750 0 : Address argAddr(temp, ArgumentsData::offsetOfArgs() + lir->mir()->argno() * sizeof(Value));
6751 0 : masm.loadValue(argAddr, out);
6752 : #ifdef DEBUG
6753 0 : Label success;
6754 0 : masm.branchTestMagic(Assembler::NotEqual, out, &success);
6755 0 : masm.assumeUnreachable("Result from ArgumentObject shouldn't be JSVAL_TYPE_MAGIC.");
6756 0 : masm.bind(&success);
6757 : #endif
6758 0 : }
6759 :
6760 : void
6761 0 : CodeGenerator::visitSetArgumentsObjectArg(LSetArgumentsObjectArg* lir)
6762 : {
6763 0 : Register temp = ToRegister(lir->getTemp(0));
6764 0 : Register argsObj = ToRegister(lir->getArgsObject());
6765 0 : ValueOperand value = ToValue(lir, LSetArgumentsObjectArg::ValueIndex);
6766 :
6767 0 : masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()), temp);
6768 0 : Address argAddr(temp, ArgumentsData::offsetOfArgs() + lir->mir()->argno() * sizeof(Value));
6769 0 : emitPreBarrier(argAddr);
6770 : #ifdef DEBUG
6771 0 : Label success;
6772 0 : masm.branchTestMagic(Assembler::NotEqual, argAddr, &success);
6773 0 : masm.assumeUnreachable("Result in ArgumentObject shouldn't be JSVAL_TYPE_MAGIC.");
6774 0 : masm.bind(&success);
6775 : #endif
6776 0 : masm.storeValue(value, argAddr);
6777 0 : }
6778 :
6779 : void
6780 0 : CodeGenerator::visitReturnFromCtor(LReturnFromCtor* lir)
6781 : {
6782 0 : ValueOperand value = ToValue(lir, LReturnFromCtor::ValueIndex);
6783 0 : Register obj = ToRegister(lir->getObject());
6784 0 : Register output = ToRegister(lir->output());
6785 :
6786 0 : Label valueIsObject, end;
6787 :
6788 0 : masm.branchTestObject(Assembler::Equal, value, &valueIsObject);
6789 :
6790 : // Value is not an object. Return that other object.
6791 0 : masm.movePtr(obj, output);
6792 0 : masm.jump(&end);
6793 :
6794 : // Value is an object. Return unbox(Value).
6795 0 : masm.bind(&valueIsObject);
6796 0 : Register payload = masm.extractObject(value, output);
6797 0 : if (payload != output)
6798 0 : masm.movePtr(payload, output);
6799 :
6800 0 : masm.bind(&end);
6801 0 : }
6802 :
6803 : typedef bool (*BoxNonStrictThisFn)(JSContext*, HandleValue, MutableHandleValue);
6804 0 : static const VMFunction BoxNonStrictThisInfo =
6805 3 : FunctionInfo<BoxNonStrictThisFn>(BoxNonStrictThis, "BoxNonStrictThis");
6806 :
6807 : void
6808 0 : CodeGenerator::visitComputeThis(LComputeThis* lir)
6809 : {
6810 0 : ValueOperand value = ToValue(lir, LComputeThis::ValueIndex);
6811 0 : ValueOperand output = ToOutValue(lir);
6812 :
6813 0 : OutOfLineCode* ool = oolCallVM(BoxNonStrictThisInfo, lir, ArgList(value), StoreValueTo(output));
6814 :
6815 0 : masm.branchTestObject(Assembler::NotEqual, value, ool->entry());
6816 0 : masm.moveValue(value, output);
6817 0 : masm.bind(ool->rejoin());
6818 0 : }
6819 :
6820 : void
6821 8 : CodeGenerator::visitImplicitThis(LImplicitThis* lir)
6822 : {
6823 0 : pushArg(ImmGCPtr(lir->mir()->name()));
6824 0 : pushArg(ToRegister(lir->env()));
6825 0 : callVM(ImplicitThisInfo, lir);
6826 8 : }
6827 :
6828 : void
6829 0 : CodeGenerator::visitArrowNewTarget(LArrowNewTarget* lir)
6830 : {
6831 0 : Register callee = ToRegister(lir->callee());
6832 0 : ValueOperand output = ToOutValue(lir);
6833 0 : masm.loadValue(Address(callee, FunctionExtended::offsetOfArrowNewTargetSlot()), output);
6834 0 : }
6835 :
6836 : void
6837 5 : CodeGenerator::visitArrayLength(LArrayLength* lir)
6838 : {
6839 0 : Address length(ToRegister(lir->elements()), ObjectElements::offsetOfLength());
6840 0 : masm.load32(length, ToRegister(lir->output()));
6841 5 : }
6842 :
6843 : static void
6844 100 : SetLengthFromIndex(MacroAssembler& masm, const LAllocation* index, const Address& length)
6845 : {
6846 0 : if (index->isConstant()) {
6847 200 : masm.store32(Imm32(ToInt32(index) + 1), length);
6848 : } else {
6849 0 : Register newLength = ToRegister(index);
6850 0 : masm.add32(Imm32(1), newLength);
6851 0 : masm.store32(newLength, length);
6852 0 : masm.sub32(Imm32(1), newLength);
6853 : }
6854 100 : }
6855 :
6856 : void
6857 0 : CodeGenerator::visitSetArrayLength(LSetArrayLength* lir)
6858 : {
6859 0 : Address length(ToRegister(lir->elements()), ObjectElements::offsetOfLength());
6860 0 : SetLengthFromIndex(masm, lir->index(), length);
6861 0 : }
6862 :
6863 : template <class OrderedHashTable>
6864 : static void
6865 : RangeFront(MacroAssembler&, Register, Register, Register);
6866 :
6867 : template <>
6868 : void
6869 6 : RangeFront<ValueMap>(MacroAssembler& masm, Register range, Register i, Register front)
6870 : {
6871 0 : masm.loadPtr(Address(range, ValueMap::Range::offsetOfHashTable()), front);
6872 12 : masm.loadPtr(Address(front, ValueMap::offsetOfImplData()), front);
6873 :
6874 : MOZ_ASSERT(ValueMap::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0");
6875 : static_assert(ValueMap::sizeofImplData() == 24, "sizeof(Data) is 24");
6876 0 : masm.mulBy3(i, i);
6877 0 : masm.lshiftPtr(Imm32(3), i);
6878 0 : masm.addPtr(i, front);
6879 6 : }
6880 :
6881 : template <>
6882 : void
6883 6 : RangeFront<ValueSet>(MacroAssembler& masm, Register range, Register i, Register front)
6884 : {
6885 0 : masm.loadPtr(Address(range, ValueSet::Range::offsetOfHashTable()), front);
6886 12 : masm.loadPtr(Address(front, ValueSet::offsetOfImplData()), front);
6887 :
6888 : MOZ_ASSERT(ValueSet::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0");
6889 : static_assert(ValueSet::sizeofImplData() == 16, "sizeof(Data) is 16");
6890 0 : masm.lshiftPtr(Imm32(4), i);
6891 0 : masm.addPtr(i, front);
6892 6 : }
6893 :
6894 : template <class OrderedHashTable>
6895 : static void
6896 12 : RangePopFront(MacroAssembler& masm, Register range, Register front, Register dataLength,
6897 : Register temp)
6898 : {
6899 12 : Register i = temp;
6900 :
6901 36 : masm.add32(Imm32(1), Address(range, OrderedHashTable::Range::offsetOfCount()));
6902 :
6903 24 : masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), i);
6904 :
6905 0 : Label done, seek;
6906 0 : masm.bind(&seek);
6907 0 : masm.add32(Imm32(1), i);
6908 12 : masm.branch32(Assembler::AboveOrEqual, i, dataLength, &done);
6909 :
6910 : // We can add sizeof(Data) to |front| to select the next element, because
6911 : // |front| and |range.ht.data[i]| point to the same location.
6912 : MOZ_ASSERT(OrderedHashTable::offsetOfImplDataElement() == 0, "offsetof(Data, element) is 0");
6913 12 : masm.addPtr(Imm32(OrderedHashTable::sizeofImplData()), front);
6914 :
6915 36 : masm.branchTestMagic(Assembler::Equal, Address(front, OrderedHashTable::offsetOfEntryKey()),
6916 : JS_HASH_KEY_EMPTY, &seek);
6917 :
6918 0 : masm.bind(&done);
6919 0 : masm.store32(i, Address(range, OrderedHashTable::Range::offsetOfI()));
6920 12 : }
6921 :
6922 : template <class OrderedHashTable>
6923 : static inline void
6924 12 : RangeDestruct(MacroAssembler& masm, Register iter, Register range, Register temp0, Register temp1)
6925 : {
6926 0 : Register next = temp0;
6927 12 : Register prevp = temp1;
6928 :
6929 0 : masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfNext()), next);
6930 0 : masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfPrevP()), prevp);
6931 12 : masm.storePtr(next, Address(prevp, 0));
6932 :
6933 0 : Label hasNoNext;
6934 12 : masm.branchTestPtr(Assembler::Zero, next, next, &hasNoNext);
6935 :
6936 24 : masm.storePtr(prevp, Address(next, OrderedHashTable::Range::offsetOfPrevP()));
6937 :
6938 12 : masm.bind(&hasNoNext);
6939 :
6940 0 : Label nurseryAllocated;
6941 12 : masm.branchPtrInNurseryChunk(Assembler::Equal, iter, temp0, &nurseryAllocated);
6942 :
6943 12 : masm.callFreeStub(range);
6944 :
6945 0 : masm.bind(&nurseryAllocated);
6946 12 : }
6947 :
6948 : template <>
6949 : void
6950 6 : CodeGenerator::emitLoadIteratorValues<ValueMap>(Register result, Register temp, Register front)
6951 : {
6952 6 : size_t elementsOffset = NativeObject::offsetOfFixedElements();
6953 :
6954 0 : Address keyAddress(front, ValueMap::Entry::offsetOfKey());
6955 0 : Address valueAddress(front, ValueMap::Entry::offsetOfValue());
6956 0 : Address keyElemAddress(result, elementsOffset);
6957 0 : Address valueElemAddress(result, elementsOffset + sizeof(Value));
6958 0 : masm.guardedCallPreBarrier(keyElemAddress, MIRType::Value);
6959 0 : masm.guardedCallPreBarrier(valueElemAddress, MIRType::Value);
6960 0 : masm.storeValue(keyAddress, keyElemAddress, temp);
6961 12 : masm.storeValue(valueAddress, valueElemAddress, temp);
6962 :
6963 0 : Label emitBarrier, skipBarrier;
6964 0 : masm.branchValueIsNurseryCell(Assembler::Equal, keyAddress, temp, &emitBarrier);
6965 6 : masm.branchValueIsNurseryCell(Assembler::NotEqual, valueAddress, temp, &skipBarrier);
6966 : {
6967 0 : masm.bind(&emitBarrier);
6968 0 : saveVolatile(temp);
6969 0 : emitPostWriteBarrier(result);
6970 12 : restoreVolatile(temp);
6971 : }
6972 0 : masm.bind(&skipBarrier);
6973 6 : }
6974 :
6975 : template <>
6976 : void
6977 6 : CodeGenerator::emitLoadIteratorValues<ValueSet>(Register result, Register temp, Register front)
6978 : {
6979 6 : size_t elementsOffset = NativeObject::offsetOfFixedElements();
6980 :
6981 0 : Address keyAddress(front, ValueSet::offsetOfEntryKey());
6982 0 : Address keyElemAddress(result, elementsOffset);
6983 0 : masm.guardedCallPreBarrier(keyElemAddress, MIRType::Value);
6984 12 : masm.storeValue(keyAddress, keyElemAddress, temp);
6985 :
6986 0 : Label skipBarrier;
6987 6 : masm.branchValueIsNurseryCell(Assembler::NotEqual, keyAddress, temp, &skipBarrier);
6988 : {
6989 0 : saveVolatile(temp);
6990 0 : emitPostWriteBarrier(result);
6991 12 : restoreVolatile(temp);
6992 : }
6993 0 : masm.bind(&skipBarrier);
6994 6 : }
6995 :
6996 : template <class IteratorObject, class OrderedHashTable>
6997 : void
6998 12 : CodeGenerator::emitGetNextEntryForIterator(LGetNextEntryForIterator* lir)
6999 : {
7000 0 : Register iter = ToRegister(lir->iter());
7001 0 : Register result = ToRegister(lir->result());
7002 0 : Register temp = ToRegister(lir->temp0());
7003 0 : Register dataLength = ToRegister(lir->temp1());
7004 0 : Register range = ToRegister(lir->temp2());
7005 24 : Register output = ToRegister(lir->output());
7006 :
7007 : #ifdef DEBUG
7008 : // Self-hosted code is responsible for ensuring GetNextEntryForIterator is
7009 : // only called with the correct iterator class. Assert here all self-
7010 : // hosted callers of GetNextEntryForIterator perform this class check.
7011 : // No Spectre mitigations are needed because this is DEBUG-only code.
7012 0 : Label success;
7013 12 : masm.branchTestObjClassNoSpectreMitigations(Assembler::Equal, iter, &IteratorObject::class_,
7014 : temp, &success);
7015 0 : masm.assumeUnreachable("Iterator object should have the correct class.");
7016 12 : masm.bind(&success);
7017 : #endif
7018 :
7019 24 : masm.loadPrivate(Address(iter, NativeObject::getFixedSlotOffset(IteratorObject::RangeSlot)),
7020 : range);
7021 :
7022 0 : Label iterAlreadyDone, iterDone, done;
7023 12 : masm.branchTestPtr(Assembler::Zero, range, range, &iterAlreadyDone);
7024 :
7025 0 : masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), temp);
7026 0 : masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfHashTable()), dataLength);
7027 0 : masm.load32(Address(dataLength, OrderedHashTable::offsetOfImplDataLength()), dataLength);
7028 12 : masm.branch32(Assembler::AboveOrEqual, temp, dataLength, &iterDone);
7029 : {
7030 24 : masm.push(iter);
7031 :
7032 0 : Register front = iter;
7033 12 : RangeFront<OrderedHashTable>(masm, range, temp, front);
7034 :
7035 12 : emitLoadIteratorValues<OrderedHashTable>(result, temp, front);
7036 :
7037 12 : RangePopFront<OrderedHashTable>(masm, range, front, dataLength, temp);
7038 :
7039 0 : masm.pop(iter);
7040 24 : masm.move32(Imm32(0), output);
7041 : }
7042 24 : masm.jump(&done);
7043 : {
7044 12 : masm.bind(&iterDone);
7045 :
7046 12 : RangeDestruct<OrderedHashTable>(masm, iter, range, temp, dataLength);
7047 :
7048 0 : masm.storeValue(PrivateValue(nullptr),
7049 12 : Address(iter, NativeObject::getFixedSlotOffset(IteratorObject::RangeSlot)));
7050 :
7051 12 : masm.bind(&iterAlreadyDone);
7052 :
7053 24 : masm.move32(Imm32(1), output);
7054 : }
7055 0 : masm.bind(&done);
7056 12 : }
7057 :
7058 : void
7059 12 : CodeGenerator::visitGetNextEntryForIterator(LGetNextEntryForIterator* lir)
7060 : {
7061 0 : if (lir->mir()->mode() == MGetNextEntryForIterator::Map) {
7062 6 : emitGetNextEntryForIterator<MapIteratorObject, ValueMap>(lir);
7063 : } else {
7064 0 : MOZ_ASSERT(lir->mir()->mode() == MGetNextEntryForIterator::Set);
7065 6 : emitGetNextEntryForIterator<SetIteratorObject, ValueSet>(lir);
7066 : }
7067 12 : }
7068 :
7069 : void
7070 0 : CodeGenerator::emitWasmCallBase(MWasmCall* mir, bool needsBoundsCheck)
7071 : {
7072 0 : if (mir->spIncrement())
7073 0 : masm.freeStack(mir->spIncrement());
7074 :
7075 0 : MOZ_ASSERT((sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment == 0);
7076 : static_assert(WasmStackAlignment >= ABIStackAlignment &&
7077 : WasmStackAlignment % ABIStackAlignment == 0,
7078 : "The wasm stack alignment should subsume the ABI-required alignment");
7079 :
7080 : #ifdef DEBUG
7081 0 : Label ok;
7082 0 : masm.branchTestStackPtr(Assembler::Zero, Imm32(WasmStackAlignment - 1), &ok);
7083 0 : masm.breakpoint();
7084 0 : masm.bind(&ok);
7085 : #endif
7086 :
7087 : // LWasmCallBase::isCallPreserved() assumes that all MWasmCalls preserve the
7088 : // TLS and pinned regs. The only case where where we don't have to reload
7089 : // the TLS and pinned regs is when the callee preserves them.
7090 0 : bool reloadRegs = true;
7091 :
7092 0 : const wasm::CallSiteDesc& desc = mir->desc();
7093 0 : const wasm::CalleeDesc& callee = mir->callee();
7094 0 : switch (callee.which()) {
7095 : case wasm::CalleeDesc::Func:
7096 0 : masm.call(desc, callee.funcIndex());
7097 0 : reloadRegs = false;
7098 0 : break;
7099 : case wasm::CalleeDesc::Import:
7100 0 : masm.wasmCallImport(desc, callee);
7101 0 : break;
7102 : case wasm::CalleeDesc::AsmJSTable:
7103 : case wasm::CalleeDesc::WasmTable:
7104 0 : masm.wasmCallIndirect(desc, callee, needsBoundsCheck);
7105 0 : reloadRegs = callee.which() == wasm::CalleeDesc::WasmTable && callee.wasmTableIsExternal();
7106 : break;
7107 : case wasm::CalleeDesc::Builtin:
7108 0 : masm.call(desc, callee.builtin());
7109 0 : reloadRegs = false;
7110 0 : break;
7111 : case wasm::CalleeDesc::BuiltinInstanceMethod:
7112 0 : masm.wasmCallBuiltinInstanceMethod(desc, mir->instanceArg(), callee.builtin());
7113 0 : break;
7114 : }
7115 :
7116 0 : if (reloadRegs) {
7117 0 : masm.loadWasmTlsRegFromFrame();
7118 0 : masm.loadWasmPinnedRegsFromTls();
7119 : }
7120 :
7121 0 : if (mir->spIncrement())
7122 0 : masm.reserveStack(mir->spIncrement());
7123 0 : }
7124 :
7125 : void
7126 0 : CodeGenerator::visitWasmCall(LWasmCall* ins)
7127 : {
7128 0 : emitWasmCallBase(ins->mir(), ins->needsBoundsCheck());
7129 0 : }
7130 :
7131 : void
7132 0 : CodeGenerator::visitWasmCallVoid(LWasmCallVoid* ins)
7133 : {
7134 0 : emitWasmCallBase(ins->mir(), ins->needsBoundsCheck());
7135 0 : }
7136 :
7137 : void
7138 0 : CodeGenerator::visitWasmCallI64(LWasmCallI64* ins)
7139 : {
7140 0 : emitWasmCallBase(ins->mir(), ins->needsBoundsCheck());
7141 0 : }
7142 :
7143 : void
7144 0 : CodeGenerator::visitWasmLoadGlobalVar(LWasmLoadGlobalVar* ins)
7145 : {
7146 0 : MWasmLoadGlobalVar* mir = ins->mir();
7147 :
7148 0 : MIRType type = mir->type();
7149 0 : MOZ_ASSERT(IsNumberType(type) || IsSimdType(type));
7150 :
7151 0 : Register tls = ToRegister(ins->tlsPtr());
7152 0 : Address addr(tls, offsetof(wasm::TlsData, globalArea) + mir->globalDataOffset());
7153 0 : if (mir->isIndirect()) {
7154 0 : Register tmp = ToRegister(ins->addrTemp());
7155 0 : masm.loadPtr(addr, tmp);
7156 0 : addr = Address(tmp, 0);
7157 : }
7158 0 : switch (type) {
7159 : case MIRType::Int32:
7160 0 : masm.load32(addr, ToRegister(ins->output()));
7161 0 : break;
7162 : case MIRType::Float32:
7163 0 : masm.loadFloat32(addr, ToFloatRegister(ins->output()));
7164 : break;
7165 : case MIRType::Double:
7166 0 : masm.loadDouble(addr, ToFloatRegister(ins->output()));
7167 : break;
7168 : // Aligned access: code is aligned on PageSize + there is padding
7169 : // before the global data section.
7170 : case MIRType::Int8x16:
7171 : case MIRType::Int16x8:
7172 : case MIRType::Int32x4:
7173 : case MIRType::Bool8x16:
7174 : case MIRType::Bool16x8:
7175 : case MIRType::Bool32x4:
7176 0 : masm.loadInt32x4(addr, ToFloatRegister(ins->output()));
7177 0 : break;
7178 : case MIRType::Float32x4:
7179 0 : masm.loadFloat32x4(addr, ToFloatRegister(ins->output()));
7180 0 : break;
7181 : default:
7182 0 : MOZ_CRASH("unexpected type in visitWasmLoadGlobalVar");
7183 : }
7184 0 : }
7185 :
7186 : void
7187 0 : CodeGenerator::visitWasmStoreGlobalVar(LWasmStoreGlobalVar* ins)
7188 : {
7189 0 : MWasmStoreGlobalVar* mir = ins->mir();
7190 :
7191 0 : MIRType type = mir->value()->type();
7192 0 : MOZ_ASSERT(IsNumberType(type) || IsSimdType(type));
7193 :
7194 0 : Register tls = ToRegister(ins->tlsPtr());
7195 0 : Address addr(tls, offsetof(wasm::TlsData, globalArea) + mir->globalDataOffset());
7196 0 : if (mir->isIndirect()) {
7197 0 : Register tmp = ToRegister(ins->addrTemp());
7198 0 : masm.loadPtr(addr, tmp);
7199 0 : addr = Address(tmp, 0);
7200 : }
7201 0 : switch (type) {
7202 : case MIRType::Int32:
7203 0 : masm.store32(ToRegister(ins->value()), addr);
7204 0 : break;
7205 : case MIRType::Float32:
7206 0 : masm.storeFloat32(ToFloatRegister(ins->value()), addr);
7207 : break;
7208 : case MIRType::Double:
7209 0 : masm.storeDouble(ToFloatRegister(ins->value()), addr);
7210 : break;
7211 : // Aligned access: code is aligned on PageSize + there is padding
7212 : // before the global data section.
7213 : case MIRType::Int8x16:
7214 : case MIRType::Int16x8:
7215 : case MIRType::Int32x4:
7216 : case MIRType::Bool8x16:
7217 : case MIRType::Bool16x8:
7218 : case MIRType::Bool32x4:
7219 0 : masm.storeInt32x4(ToFloatRegister(ins->value()), addr);
7220 0 : break;
7221 : case MIRType::Float32x4:
7222 0 : masm.storeFloat32x4(ToFloatRegister(ins->value()), addr);
7223 0 : break;
7224 : default:
7225 0 : MOZ_CRASH("unexpected type in visitWasmStoreGlobalVar");
7226 : }
7227 0 : }
7228 :
7229 : void
7230 0 : CodeGenerator::visitWasmLoadGlobalVarI64(LWasmLoadGlobalVarI64* ins)
7231 : {
7232 0 : MWasmLoadGlobalVar* mir = ins->mir();
7233 0 : MOZ_ASSERT(mir->type() == MIRType::Int64);
7234 :
7235 0 : Register tls = ToRegister(ins->tlsPtr());
7236 0 : Address addr(tls, offsetof(wasm::TlsData, globalArea) + mir->globalDataOffset());
7237 :
7238 0 : Register64 output = ToOutRegister64(ins);
7239 0 : if (mir->isIndirect()) {
7240 0 : Register tmp = ToRegister(ins->addrTemp());
7241 0 : masm.loadPtr(addr, tmp);
7242 0 : addr = Address(tmp, 0);
7243 : }
7244 0 : masm.load64(addr, output);
7245 0 : }
7246 :
7247 : void
7248 0 : CodeGenerator::visitWasmStoreGlobalVarI64(LWasmStoreGlobalVarI64* ins)
7249 : {
7250 0 : MWasmStoreGlobalVar* mir = ins->mir();
7251 0 : MOZ_ASSERT(mir->value()->type() == MIRType::Int64);
7252 :
7253 0 : Register tls = ToRegister(ins->tlsPtr());
7254 0 : Address addr(tls, offsetof(wasm::TlsData, globalArea) + mir->globalDataOffset());
7255 :
7256 0 : Register64 value = ToRegister64(ins->value());
7257 0 : if (mir->isIndirect()) {
7258 0 : Register tmp = ToRegister(ins->addrTemp());
7259 0 : masm.loadPtr(addr, tmp);
7260 0 : addr = Address(tmp, 0);
7261 : }
7262 0 : masm.store64(value, addr);
7263 0 : }
7264 :
7265 : void
7266 0 : CodeGenerator::visitTypedArrayLength(LTypedArrayLength* lir)
7267 : {
7268 0 : Register obj = ToRegister(lir->object());
7269 0 : Register out = ToRegister(lir->output());
7270 0 : masm.unboxInt32(Address(obj, TypedArrayObject::lengthOffset()), out);
7271 0 : }
7272 :
7273 : void
7274 0 : CodeGenerator::visitTypedArrayElements(LTypedArrayElements* lir)
7275 : {
7276 0 : Register obj = ToRegister(lir->object());
7277 0 : Register out = ToRegister(lir->output());
7278 0 : masm.loadPtr(Address(obj, TypedArrayObject::dataOffset()), out);
7279 0 : }
7280 :
7281 : void
7282 0 : CodeGenerator::visitSetDisjointTypedElements(LSetDisjointTypedElements* lir)
7283 : {
7284 0 : Register target = ToRegister(lir->target());
7285 0 : Register targetOffset = ToRegister(lir->targetOffset());
7286 0 : Register source = ToRegister(lir->source());
7287 :
7288 0 : Register temp = ToRegister(lir->temp());
7289 :
7290 0 : masm.setupUnalignedABICall(temp);
7291 0 : masm.passABIArg(target);
7292 0 : masm.passABIArg(targetOffset);
7293 0 : masm.passABIArg(source);
7294 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::SetDisjointTypedElements));
7295 0 : }
7296 :
7297 : void
7298 0 : CodeGenerator::visitTypedObjectDescr(LTypedObjectDescr* lir)
7299 : {
7300 0 : Register obj = ToRegister(lir->object());
7301 0 : Register out = ToRegister(lir->output());
7302 0 : masm.loadTypedObjectDescr(obj, out);
7303 0 : }
7304 :
7305 : void
7306 0 : CodeGenerator::visitTypedObjectElements(LTypedObjectElements* lir)
7307 : {
7308 0 : Register obj = ToRegister(lir->object());
7309 0 : Register out = ToRegister(lir->output());
7310 :
7311 0 : if (lir->mir()->definitelyOutline()) {
7312 0 : masm.loadPtr(Address(obj, OutlineTypedObject::offsetOfData()), out);
7313 : } else {
7314 0 : Label inlineObject, done;
7315 0 : masm.branchIfInlineTypedObject(obj, out, &inlineObject);
7316 :
7317 0 : masm.loadPtr(Address(obj, OutlineTypedObject::offsetOfData()), out);
7318 0 : masm.jump(&done);
7319 :
7320 0 : masm.bind(&inlineObject);
7321 0 : masm.computeEffectiveAddress(Address(obj, InlineTypedObject::offsetOfDataStart()), out);
7322 0 : masm.bind(&done);
7323 : }
7324 0 : }
7325 :
7326 : void
7327 0 : CodeGenerator::visitSetTypedObjectOffset(LSetTypedObjectOffset* lir)
7328 : {
7329 0 : Register object = ToRegister(lir->object());
7330 0 : Register offset = ToRegister(lir->offset());
7331 0 : Register temp0 = ToRegister(lir->temp0());
7332 0 : Register temp1 = ToRegister(lir->temp1());
7333 :
7334 : // Compute the base pointer for the typed object's owner.
7335 0 : masm.loadPtr(Address(object, OutlineTypedObject::offsetOfOwner()), temp0);
7336 :
7337 0 : Label inlineObject, done;
7338 0 : masm.branchIfInlineTypedObject(temp0, temp1, &inlineObject);
7339 :
7340 0 : masm.loadPrivate(Address(temp0, ArrayBufferObject::offsetOfDataSlot()), temp0);
7341 0 : masm.jump(&done);
7342 :
7343 0 : masm.bind(&inlineObject);
7344 0 : masm.addPtr(ImmWord(InlineTypedObject::offsetOfDataStart()), temp0);
7345 :
7346 0 : masm.bind(&done);
7347 :
7348 : // Compute the new data pointer and set it in the object.
7349 0 : masm.addPtr(offset, temp0);
7350 0 : masm.storePtr(temp0, Address(object, OutlineTypedObject::offsetOfData()));
7351 0 : }
7352 :
7353 : void
7354 3 : CodeGenerator::visitStringLength(LStringLength* lir)
7355 : {
7356 0 : Register input = ToRegister(lir->string());
7357 6 : Register output = ToRegister(lir->output());
7358 :
7359 0 : masm.loadStringLength(input, output);
7360 3 : }
7361 :
7362 : void
7363 42 : CodeGenerator::visitMinMaxI(LMinMaxI* ins)
7364 : {
7365 0 : Register first = ToRegister(ins->first());
7366 126 : Register output = ToRegister(ins->output());
7367 :
7368 42 : MOZ_ASSERT(first == output);
7369 :
7370 0 : Label done;
7371 0 : Assembler::Condition cond = ins->mir()->isMax()
7372 0 : ? Assembler::GreaterThan
7373 42 : : Assembler::LessThan;
7374 :
7375 0 : if (ins->second()->isConstant()) {
7376 0 : masm.branch32(cond, first, Imm32(ToInt32(ins->second())), &done);
7377 126 : masm.move32(Imm32(ToInt32(ins->second())), output);
7378 : } else {
7379 0 : masm.branch32(cond, first, ToRegister(ins->second()), &done);
7380 0 : masm.move32(ToRegister(ins->second()), output);
7381 : }
7382 :
7383 0 : masm.bind(&done);
7384 42 : }
7385 :
7386 : void
7387 0 : CodeGenerator::visitAbsI(LAbsI* ins)
7388 : {
7389 0 : Register input = ToRegister(ins->input());
7390 0 : Label positive;
7391 :
7392 0 : MOZ_ASSERT(input == ToRegister(ins->output()));
7393 0 : masm.branchTest32(Assembler::NotSigned, input, input, &positive);
7394 0 : masm.neg32(input);
7395 0 : LSnapshot* snapshot = ins->snapshot();
7396 : #if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
7397 : if (snapshot)
7398 : bailoutCmp32(Assembler::Equal, input, Imm32(INT32_MIN), snapshot);
7399 : #else
7400 0 : if (snapshot)
7401 0 : bailoutIf(Assembler::Overflow, snapshot);
7402 : #endif
7403 0 : masm.bind(&positive);
7404 0 : }
7405 :
7406 : void
7407 0 : CodeGenerator::visitPowI(LPowI* ins)
7408 : {
7409 0 : FloatRegister value = ToFloatRegister(ins->value());
7410 0 : Register power = ToRegister(ins->power());
7411 0 : Register temp = ToRegister(ins->temp());
7412 :
7413 0 : MOZ_ASSERT(power != temp);
7414 :
7415 0 : masm.setupUnalignedABICall(temp);
7416 0 : masm.passABIArg(value, MoveOp::DOUBLE);
7417 0 : masm.passABIArg(power);
7418 :
7419 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::powi), MoveOp::DOUBLE);
7420 0 : MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
7421 0 : }
7422 :
7423 : void
7424 0 : CodeGenerator::visitPowD(LPowD* ins)
7425 : {
7426 0 : FloatRegister value = ToFloatRegister(ins->value());
7427 0 : FloatRegister power = ToFloatRegister(ins->power());
7428 0 : Register temp = ToRegister(ins->temp());
7429 :
7430 0 : masm.setupUnalignedABICall(temp);
7431 0 : masm.passABIArg(value, MoveOp::DOUBLE);
7432 0 : masm.passABIArg(power, MoveOp::DOUBLE);
7433 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ecmaPow), MoveOp::DOUBLE);
7434 :
7435 0 : MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
7436 0 : }
7437 :
7438 : using PowFn = bool (*)(JSContext*, HandleValue, HandleValue, MutableHandleValue);
7439 0 : static const VMFunction PowInfo =
7440 3 : FunctionInfo<PowFn>(js::math_pow_handle, "math_pow_handle");
7441 :
7442 : void
7443 0 : CodeGenerator::visitPowV(LPowV* ins)
7444 : {
7445 0 : pushArg(ToValue(ins, LPowV::PowerInput));
7446 0 : pushArg(ToValue(ins, LPowV::ValueInput));
7447 0 : callVM(PowInfo, ins);
7448 0 : }
7449 :
7450 : void
7451 0 : CodeGenerator::visitSignI(LSignI* ins)
7452 : {
7453 0 : Register input = ToRegister(ins->input());
7454 0 : Register output = ToRegister(ins->output());
7455 :
7456 0 : Label done;
7457 0 : masm.move32(input, output);
7458 0 : masm.rshift32Arithmetic(Imm32(31), output);
7459 0 : masm.branch32(Assembler::LessThanOrEqual, input, Imm32(0), &done);
7460 0 : masm.move32(Imm32(1), output);
7461 0 : masm.bind(&done);
7462 0 : }
7463 :
7464 : void
7465 0 : CodeGenerator::visitSignD(LSignD* ins)
7466 : {
7467 0 : FloatRegister input = ToFloatRegister(ins->input());
7468 0 : FloatRegister output = ToFloatRegister(ins->output());
7469 :
7470 0 : Label done, zeroOrNaN, negative;
7471 0 : masm.loadConstantDouble(0.0, output);
7472 0 : masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, output, &zeroOrNaN);
7473 0 : masm.branchDouble(Assembler::DoubleLessThan, input, output, &negative);
7474 :
7475 0 : masm.loadConstantDouble(1.0, output);
7476 0 : masm.jump(&done);
7477 :
7478 0 : masm.bind(&negative);
7479 0 : masm.loadConstantDouble(-1.0, output);
7480 0 : masm.jump(&done);
7481 :
7482 0 : masm.bind(&zeroOrNaN);
7483 0 : masm.moveDouble(input, output);
7484 :
7485 0 : masm.bind(&done);
7486 0 : }
7487 :
7488 : void
7489 0 : CodeGenerator::visitSignDI(LSignDI* ins)
7490 : {
7491 0 : FloatRegister input = ToFloatRegister(ins->input());
7492 0 : FloatRegister temp = ToFloatRegister(ins->temp());
7493 0 : Register output = ToRegister(ins->output());
7494 :
7495 0 : Label done, zeroOrNaN, negative;
7496 0 : masm.loadConstantDouble(0.0, temp);
7497 0 : masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, temp, &zeroOrNaN);
7498 0 : masm.branchDouble(Assembler::DoubleLessThan, input, temp, &negative);
7499 :
7500 0 : masm.move32(Imm32(1), output);
7501 0 : masm.jump(&done);
7502 :
7503 0 : masm.bind(&negative);
7504 0 : masm.move32(Imm32(-1), output);
7505 0 : masm.jump(&done);
7506 :
7507 : // Bailout for NaN and negative zero.
7508 0 : Label bailout;
7509 0 : masm.bind(&zeroOrNaN);
7510 0 : masm.branchDouble(Assembler::DoubleUnordered, input, input, &bailout);
7511 :
7512 : // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0
7513 : // is -Infinity instead of Infinity.
7514 0 : Label isNegInf;
7515 0 : masm.loadConstantDouble(1.0, temp);
7516 0 : masm.divDouble(input, temp);
7517 0 : masm.branchDouble(Assembler::DoubleLessThan, temp, input, &bailout);
7518 0 : masm.move32(Imm32(0), output);
7519 :
7520 0 : bailoutFrom(&bailout, ins->snapshot());
7521 :
7522 0 : masm.bind(&done);
7523 0 : }
7524 :
7525 : void
7526 0 : CodeGenerator::visitMathFunctionD(LMathFunctionD* ins)
7527 : {
7528 0 : Register temp = ToRegister(ins->temp());
7529 0 : FloatRegister input = ToFloatRegister(ins->input());
7530 0 : MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
7531 :
7532 0 : masm.setupUnalignedABICall(temp);
7533 :
7534 0 : const MathCache* mathCache = ins->mir()->cache();
7535 0 : if (mathCache) {
7536 0 : masm.movePtr(ImmPtr(mathCache), temp);
7537 0 : masm.passABIArg(temp);
7538 : }
7539 0 : masm.passABIArg(input, MoveOp::DOUBLE);
7540 :
7541 : # define MAYBE_CACHED(fcn) (mathCache ? (void*)fcn ## _impl : (void*)fcn ## _uncached)
7542 :
7543 0 : void* funptr = nullptr;
7544 0 : switch (ins->mir()->function()) {
7545 : case MMathFunction::Log:
7546 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log));
7547 : break;
7548 : case MMathFunction::Sin:
7549 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_sin));
7550 : break;
7551 : case MMathFunction::Cos:
7552 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cos));
7553 : break;
7554 : case MMathFunction::Exp:
7555 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_exp));
7556 : break;
7557 : case MMathFunction::Tan:
7558 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_tan));
7559 : break;
7560 : case MMathFunction::ATan:
7561 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_atan));
7562 : break;
7563 : case MMathFunction::ASin:
7564 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_asin));
7565 : break;
7566 : case MMathFunction::ACos:
7567 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_acos));
7568 : break;
7569 : case MMathFunction::Log10:
7570 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log10));
7571 : break;
7572 : case MMathFunction::Log2:
7573 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log2));
7574 : break;
7575 : case MMathFunction::Log1P:
7576 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_log1p));
7577 : break;
7578 : case MMathFunction::ExpM1:
7579 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_expm1));
7580 : break;
7581 : case MMathFunction::CosH:
7582 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cosh));
7583 : break;
7584 : case MMathFunction::SinH:
7585 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_sinh));
7586 : break;
7587 : case MMathFunction::TanH:
7588 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_tanh));
7589 : break;
7590 : case MMathFunction::ACosH:
7591 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_acosh));
7592 : break;
7593 : case MMathFunction::ASinH:
7594 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_asinh));
7595 : break;
7596 : case MMathFunction::ATanH:
7597 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_atanh));
7598 : break;
7599 : case MMathFunction::Trunc:
7600 : funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_trunc_uncached);
7601 : break;
7602 : case MMathFunction::Cbrt:
7603 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED(js::math_cbrt));
7604 : break;
7605 : case MMathFunction::Floor:
7606 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_floor_impl);
7607 0 : break;
7608 : case MMathFunction::Ceil:
7609 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_ceil_impl);
7610 0 : break;
7611 : case MMathFunction::Round:
7612 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, js::math_round_impl);
7613 0 : break;
7614 : default:
7615 0 : MOZ_CRASH("Unknown math function");
7616 : }
7617 :
7618 : # undef MAYBE_CACHED
7619 :
7620 0 : masm.callWithABI(funptr, MoveOp::DOUBLE);
7621 0 : }
7622 :
7623 : void
7624 0 : CodeGenerator::visitMathFunctionF(LMathFunctionF* ins)
7625 : {
7626 0 : Register temp = ToRegister(ins->temp());
7627 0 : FloatRegister input = ToFloatRegister(ins->input());
7628 0 : MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnFloat32Reg);
7629 :
7630 0 : masm.setupUnalignedABICall(temp);
7631 0 : masm.passABIArg(input, MoveOp::FLOAT32);
7632 :
7633 0 : void* funptr = nullptr;
7634 0 : CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check;
7635 0 : switch (ins->mir()->function()) {
7636 : case MMathFunction::Floor:
7637 : funptr = JS_FUNC_TO_DATA_PTR(void*, floorf);
7638 : check = CheckUnsafeCallWithABI::DontCheckOther;
7639 : break;
7640 : case MMathFunction::Round:
7641 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, math_roundf_impl);
7642 0 : break;
7643 : case MMathFunction::Trunc:
7644 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, math_truncf_impl);
7645 0 : break;
7646 : case MMathFunction::Ceil:
7647 0 : funptr = JS_FUNC_TO_DATA_PTR(void*, ceilf);
7648 0 : check = CheckUnsafeCallWithABI::DontCheckOther;
7649 0 : break;
7650 : default:
7651 0 : MOZ_CRASH("Unknown or unsupported float32 math function");
7652 : }
7653 :
7654 0 : masm.callWithABI(funptr, MoveOp::FLOAT32, check);
7655 0 : }
7656 :
7657 : void
7658 0 : CodeGenerator::visitModD(LModD* ins)
7659 : {
7660 0 : FloatRegister lhs = ToFloatRegister(ins->lhs());
7661 0 : FloatRegister rhs = ToFloatRegister(ins->rhs());
7662 :
7663 0 : MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
7664 0 : MOZ_ASSERT(ins->temp()->isBogusTemp() == gen->compilingWasm());
7665 :
7666 0 : if (gen->compilingWasm()) {
7667 0 : masm.setupWasmABICall();
7668 0 : masm.passABIArg(lhs, MoveOp::DOUBLE);
7669 0 : masm.passABIArg(rhs, MoveOp::DOUBLE);
7670 0 : masm.callWithABI(ins->mir()->bytecodeOffset(), wasm::SymbolicAddress::ModD, MoveOp::DOUBLE);
7671 : } else {
7672 0 : masm.setupUnalignedABICall(ToRegister(ins->temp()));
7673 0 : masm.passABIArg(lhs, MoveOp::DOUBLE);
7674 0 : masm.passABIArg(rhs, MoveOp::DOUBLE);
7675 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, NumberMod), MoveOp::DOUBLE);
7676 : }
7677 0 : }
7678 :
7679 : typedef bool (*BinaryFn)(JSContext*, MutableHandleValue, MutableHandleValue, MutableHandleValue);
7680 :
7681 0 : static const VMFunction AddInfo = FunctionInfo<BinaryFn>(js::AddValues, "AddValues");
7682 0 : static const VMFunction SubInfo = FunctionInfo<BinaryFn>(js::SubValues, "SubValues");
7683 0 : static const VMFunction MulInfo = FunctionInfo<BinaryFn>(js::MulValues, "MulValues");
7684 0 : static const VMFunction DivInfo = FunctionInfo<BinaryFn>(js::DivValues, "DivValues");
7685 0 : static const VMFunction ModInfo = FunctionInfo<BinaryFn>(js::ModValues, "ModValues");
7686 2 : static const VMFunction UrshInfo = FunctionInfo<BinaryFn>(js::UrshValues, "UrshValues");
7687 :
7688 : void
7689 0 : CodeGenerator::visitBinaryV(LBinaryV* lir)
7690 : {
7691 0 : pushArg(ToValue(lir, LBinaryV::RhsInput));
7692 0 : pushArg(ToValue(lir, LBinaryV::LhsInput));
7693 :
7694 0 : switch (lir->jsop()) {
7695 : case JSOP_ADD:
7696 0 : callVM(AddInfo, lir);
7697 0 : break;
7698 :
7699 : case JSOP_SUB:
7700 0 : callVM(SubInfo, lir);
7701 0 : break;
7702 :
7703 : case JSOP_MUL:
7704 0 : callVM(MulInfo, lir);
7705 0 : break;
7706 :
7707 : case JSOP_DIV:
7708 0 : callVM(DivInfo, lir);
7709 0 : break;
7710 :
7711 : case JSOP_MOD:
7712 0 : callVM(ModInfo, lir);
7713 0 : break;
7714 :
7715 : case JSOP_URSH:
7716 0 : callVM(UrshInfo, lir);
7717 0 : break;
7718 :
7719 : default:
7720 0 : MOZ_CRASH("Unexpected binary op");
7721 : }
7722 0 : }
7723 :
7724 : void
7725 0 : CodeGenerator::emitCompareS(LInstruction* lir, JSOp op, Register left, Register right,
7726 : Register output)
7727 : {
7728 49 : MOZ_ASSERT(lir->isCompareS() || lir->isCompareStrictS());
7729 :
7730 0 : OutOfLineCode* ool = nullptr;
7731 :
7732 23 : if (op == JSOP_EQ || op == JSOP_STRICTEQ) {
7733 0 : ool = oolCallVM(StringsEqualInfo, lir, ArgList(left, right), StoreRegisterTo(output));
7734 : } else {
7735 0 : MOZ_ASSERT(op == JSOP_NE || op == JSOP_STRICTNE);
7736 20 : ool = oolCallVM(StringsNotEqualInfo, lir, ArgList(left, right), StoreRegisterTo(output));
7737 : }
7738 :
7739 23 : masm.compareStrings(op, left, right, output, ool->entry());
7740 :
7741 0 : masm.bind(ool->rejoin());
7742 23 : }
7743 :
7744 : void
7745 3 : CodeGenerator::visitCompareStrictS(LCompareStrictS* lir)
7746 : {
7747 0 : JSOp op = lir->mir()->jsop();
7748 3 : MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE);
7749 :
7750 0 : const ValueOperand leftV = ToValue(lir, LCompareStrictS::Lhs);
7751 6 : Register right = ToRegister(lir->right());
7752 0 : Register output = ToRegister(lir->output());
7753 :
7754 9 : Label string, done;
7755 :
7756 0 : masm.branchTestString(Assembler::Equal, leftV, &string);
7757 0 : masm.move32(Imm32(op == JSOP_STRICTNE), output);
7758 6 : masm.jump(&done);
7759 :
7760 3 : masm.bind(&string);
7761 : #ifdef JS_NUNBOX32
7762 : Register left = leftV.payloadReg();
7763 : #else
7764 6 : Register left = ToTempUnboxRegister(lir->tempToUnbox());
7765 : #endif
7766 6 : masm.unboxString(leftV, left);
7767 3 : emitCompareS(lir, op, left, right, output);
7768 :
7769 0 : masm.bind(&done);
7770 3 : }
7771 :
7772 : void
7773 20 : CodeGenerator::visitCompareS(LCompareS* lir)
7774 : {
7775 0 : JSOp op = lir->mir()->jsop();
7776 40 : Register left = ToRegister(lir->left());
7777 40 : Register right = ToRegister(lir->right());
7778 0 : Register output = ToRegister(lir->output());
7779 :
7780 0 : emitCompareS(lir, op, left, right, output);
7781 0 : }
7782 :
7783 : typedef bool (*CompareFn)(JSContext*, MutableHandleValue, MutableHandleValue, bool*);
7784 1 : static const VMFunction EqInfo =
7785 0 : FunctionInfo<CompareFn>(jit::LooselyEqual<true>, "LooselyEqual");
7786 0 : static const VMFunction NeInfo =
7787 3 : FunctionInfo<CompareFn>(jit::LooselyEqual<false>, "LooselyEqual");
7788 1 : static const VMFunction StrictEqInfo =
7789 0 : FunctionInfo<CompareFn>(jit::StrictlyEqual<true>, "StrictlyEqual");
7790 0 : static const VMFunction StrictNeInfo =
7791 0 : FunctionInfo<CompareFn>(jit::StrictlyEqual<false>, "StrictlyEqual");
7792 0 : static const VMFunction LtInfo =
7793 0 : FunctionInfo<CompareFn>(jit::LessThan, "LessThan");
7794 0 : static const VMFunction LeInfo =
7795 0 : FunctionInfo<CompareFn>(jit::LessThanOrEqual, "LessThanOrEqual");
7796 0 : static const VMFunction GtInfo =
7797 0 : FunctionInfo<CompareFn>(jit::GreaterThan, "GreaterThan");
7798 0 : static const VMFunction GeInfo =
7799 0 : FunctionInfo<CompareFn>(jit::GreaterThanOrEqual, "GreaterThanOrEqual");
7800 :
7801 : void
7802 0 : CodeGenerator::visitCompareVM(LCompareVM* lir)
7803 : {
7804 0 : pushArg(ToValue(lir, LBinaryV::RhsInput));
7805 0 : pushArg(ToValue(lir, LBinaryV::LhsInput));
7806 :
7807 0 : switch (lir->mir()->jsop()) {
7808 : case JSOP_EQ:
7809 0 : callVM(EqInfo, lir);
7810 0 : break;
7811 :
7812 : case JSOP_NE:
7813 0 : callVM(NeInfo, lir);
7814 0 : break;
7815 :
7816 : case JSOP_STRICTEQ:
7817 0 : callVM(StrictEqInfo, lir);
7818 0 : break;
7819 :
7820 : case JSOP_STRICTNE:
7821 0 : callVM(StrictNeInfo, lir);
7822 0 : break;
7823 :
7824 : case JSOP_LT:
7825 0 : callVM(LtInfo, lir);
7826 0 : break;
7827 :
7828 : case JSOP_LE:
7829 0 : callVM(LeInfo, lir);
7830 0 : break;
7831 :
7832 : case JSOP_GT:
7833 0 : callVM(GtInfo, lir);
7834 0 : break;
7835 :
7836 : case JSOP_GE:
7837 0 : callVM(GeInfo, lir);
7838 0 : break;
7839 :
7840 : default:
7841 0 : MOZ_CRASH("Unexpected compare op");
7842 : }
7843 0 : }
7844 :
7845 : void
7846 1 : CodeGenerator::visitIsNullOrLikeUndefinedV(LIsNullOrLikeUndefinedV* lir)
7847 : {
7848 0 : JSOp op = lir->mir()->jsop();
7849 2 : MCompare::CompareType compareType = lir->mir()->compareType();
7850 1 : MOZ_ASSERT(compareType == MCompare::Compare_Undefined ||
7851 : compareType == MCompare::Compare_Null);
7852 :
7853 0 : const ValueOperand value = ToValue(lir, LIsNullOrLikeUndefinedV::Value);
7854 0 : Register output = ToRegister(lir->output());
7855 :
7856 1 : if (op == JSOP_EQ || op == JSOP_NE) {
7857 0 : MOZ_ASSERT(lir->mir()->lhs()->type() != MIRType::Object ||
7858 : lir->mir()->operandMightEmulateUndefined(),
7859 : "Operands which can't emulate undefined should have been folded");
7860 :
7861 0 : OutOfLineTestObjectWithLabels* ool = nullptr;
7862 0 : Maybe<Label> label1, label2;
7863 : Label* nullOrLikeUndefined;
7864 : Label* notNullOrLikeUndefined;
7865 0 : if (lir->mir()->operandMightEmulateUndefined()) {
7866 0 : ool = new(alloc()) OutOfLineTestObjectWithLabels();
7867 0 : addOutOfLineCode(ool, lir->mir());
7868 0 : nullOrLikeUndefined = ool->label1();
7869 0 : notNullOrLikeUndefined = ool->label2();
7870 : } else {
7871 0 : label1.emplace();
7872 0 : label2.emplace();
7873 0 : nullOrLikeUndefined = label1.ptr();
7874 0 : notNullOrLikeUndefined = label2.ptr();
7875 : }
7876 :
7877 : {
7878 0 : ScratchTagScope tag(masm, value);
7879 0 : masm.splitTagForTest(value, tag);
7880 :
7881 0 : MDefinition* input = lir->mir()->lhs();
7882 0 : if (input->mightBeType(MIRType::Null))
7883 0 : masm.branchTestNull(Assembler::Equal, tag, nullOrLikeUndefined);
7884 0 : if (input->mightBeType(MIRType::Undefined))
7885 0 : masm.branchTestUndefined(Assembler::Equal, tag, nullOrLikeUndefined);
7886 :
7887 0 : if (ool) {
7888 : // Check whether it's a truthy object or a falsy object that emulates
7889 : // undefined.
7890 0 : masm.branchTestObject(Assembler::NotEqual, tag, notNullOrLikeUndefined);
7891 :
7892 0 : ScratchTagScopeRelease _(&tag);
7893 :
7894 0 : Register objreg = masm.extractObject(value, ToTempUnboxRegister(lir->tempToUnbox()));
7895 0 : branchTestObjectEmulatesUndefined(objreg, nullOrLikeUndefined, notNullOrLikeUndefined,
7896 0 : ToRegister(lir->temp()), ool);
7897 : // fall through
7898 : }
7899 : }
7900 :
7901 0 : Label done;
7902 :
7903 : // It's not null or undefined, and if it's an object it doesn't
7904 : // emulate undefined, so it's not like undefined.
7905 0 : masm.move32(Imm32(op == JSOP_NE), output);
7906 0 : masm.jump(&done);
7907 :
7908 0 : masm.bind(nullOrLikeUndefined);
7909 0 : masm.move32(Imm32(op == JSOP_EQ), output);
7910 :
7911 : // Both branches meet here.
7912 0 : masm.bind(&done);
7913 : return;
7914 : }
7915 :
7916 1 : MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE);
7917 :
7918 1 : Assembler::Condition cond = JSOpToCondition(compareType, op);
7919 1 : if (compareType == MCompare::Compare_Null)
7920 0 : masm.testNullSet(cond, value, output);
7921 : else
7922 1 : masm.testUndefinedSet(cond, value, output);
7923 : }
7924 :
7925 : void
7926 10 : CodeGenerator::visitIsNullOrLikeUndefinedAndBranchV(LIsNullOrLikeUndefinedAndBranchV* lir)
7927 : {
7928 10 : JSOp op = lir->cmpMir()->jsop();
7929 10 : MCompare::CompareType compareType = lir->cmpMir()->compareType();
7930 10 : MOZ_ASSERT(compareType == MCompare::Compare_Undefined ||
7931 : compareType == MCompare::Compare_Null);
7932 :
7933 0 : const ValueOperand value = ToValue(lir, LIsNullOrLikeUndefinedAndBranchV::Value);
7934 :
7935 0 : if (op == JSOP_EQ || op == JSOP_NE) {
7936 : MBasicBlock* ifTrue;
7937 : MBasicBlock* ifFalse;
7938 :
7939 0 : if (op == JSOP_EQ) {
7940 0 : ifTrue = lir->ifTrue();
7941 0 : ifFalse = lir->ifFalse();
7942 : } else {
7943 : // Swap branches.
7944 0 : ifTrue = lir->ifFalse();
7945 0 : ifFalse = lir->ifTrue();
7946 0 : op = JSOP_EQ;
7947 : }
7948 :
7949 0 : MOZ_ASSERT(lir->cmpMir()->lhs()->type() != MIRType::Object ||
7950 : lir->cmpMir()->operandMightEmulateUndefined(),
7951 : "Operands which can't emulate undefined should have been folded");
7952 :
7953 0 : OutOfLineTestObject* ool = nullptr;
7954 0 : if (lir->cmpMir()->operandMightEmulateUndefined()) {
7955 0 : ool = new(alloc()) OutOfLineTestObject();
7956 0 : addOutOfLineCode(ool, lir->cmpMir());
7957 : }
7958 :
7959 : {
7960 0 : ScratchTagScope tag(masm, value);
7961 0 : masm.splitTagForTest(value, tag);
7962 :
7963 0 : Label* ifTrueLabel = getJumpLabelForBranch(ifTrue);
7964 0 : Label* ifFalseLabel = getJumpLabelForBranch(ifFalse);
7965 :
7966 0 : MDefinition* input = lir->cmpMir()->lhs();
7967 0 : if (input->mightBeType(MIRType::Null))
7968 0 : masm.branchTestNull(Assembler::Equal, tag, ifTrueLabel);
7969 0 : if (input->mightBeType(MIRType::Undefined))
7970 0 : masm.branchTestUndefined(Assembler::Equal, tag, ifTrueLabel);
7971 :
7972 0 : if (ool) {
7973 0 : masm.branchTestObject(Assembler::NotEqual, tag, ifFalseLabel);
7974 :
7975 0 : ScratchTagScopeRelease _(&tag);
7976 :
7977 : // Objects that emulate undefined are loosely equal to null/undefined.
7978 0 : Register objreg = masm.extractObject(value, ToTempUnboxRegister(lir->tempToUnbox()));
7979 0 : Register scratch = ToRegister(lir->temp());
7980 0 : testObjectEmulatesUndefined(objreg, ifTrueLabel, ifFalseLabel, scratch, ool);
7981 : } else {
7982 0 : masm.jump(ifFalseLabel);
7983 : }
7984 : return;
7985 : }
7986 : }
7987 :
7988 10 : MOZ_ASSERT(op == JSOP_STRICTEQ || op == JSOP_STRICTNE);
7989 :
7990 10 : Assembler::Condition cond = JSOpToCondition(compareType, op);
7991 10 : if (compareType == MCompare::Compare_Null)
7992 2 : testNullEmitBranch(cond, value, lir->ifTrue(), lir->ifFalse());
7993 : else
7994 18 : testUndefinedEmitBranch(cond, value, lir->ifTrue(), lir->ifFalse());
7995 : }
7996 :
7997 : void
7998 0 : CodeGenerator::visitIsNullOrLikeUndefinedT(LIsNullOrLikeUndefinedT * lir)
7999 : {
8000 0 : MOZ_ASSERT(lir->mir()->compareType() == MCompare::Compare_Undefined ||
8001 : lir->mir()->compareType() == MCompare::Compare_Null);
8002 :
8003 0 : MIRType lhsType = lir->mir()->lhs()->type();
8004 0 : MOZ_ASSERT(lhsType == MIRType::Object || lhsType == MIRType::ObjectOrNull);
8005 :
8006 0 : JSOp op = lir->mir()->jsop();
8007 0 : MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || op == JSOP_EQ || op == JSOP_NE,
8008 : "Strict equality should have been folded");
8009 :
8010 0 : MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || lir->mir()->operandMightEmulateUndefined(),
8011 : "If the object couldn't emulate undefined, this should have been folded.");
8012 :
8013 0 : Register objreg = ToRegister(lir->input());
8014 0 : Register output = ToRegister(lir->output());
8015 :
8016 0 : if ((op == JSOP_EQ || op == JSOP_NE) && lir->mir()->operandMightEmulateUndefined()) {
8017 0 : OutOfLineTestObjectWithLabels* ool = new(alloc()) OutOfLineTestObjectWithLabels();
8018 0 : addOutOfLineCode(ool, lir->mir());
8019 :
8020 0 : Label* emulatesUndefined = ool->label1();
8021 0 : Label* doesntEmulateUndefined = ool->label2();
8022 :
8023 0 : if (lhsType == MIRType::ObjectOrNull)
8024 0 : masm.branchTestPtr(Assembler::Zero, objreg, objreg, emulatesUndefined);
8025 :
8026 : branchTestObjectEmulatesUndefined(objreg, emulatesUndefined, doesntEmulateUndefined,
8027 0 : output, ool);
8028 :
8029 0 : Label done;
8030 :
8031 0 : masm.move32(Imm32(op == JSOP_NE), output);
8032 0 : masm.jump(&done);
8033 :
8034 0 : masm.bind(emulatesUndefined);
8035 0 : masm.move32(Imm32(op == JSOP_EQ), output);
8036 0 : masm.bind(&done);
8037 : } else {
8038 0 : MOZ_ASSERT(lhsType == MIRType::ObjectOrNull);
8039 :
8040 0 : Label isNull, done;
8041 :
8042 0 : masm.branchTestPtr(Assembler::Zero, objreg, objreg, &isNull);
8043 :
8044 0 : masm.move32(Imm32(op == JSOP_NE || op == JSOP_STRICTNE), output);
8045 0 : masm.jump(&done);
8046 :
8047 0 : masm.bind(&isNull);
8048 0 : masm.move32(Imm32(op == JSOP_EQ || op == JSOP_STRICTEQ), output);
8049 :
8050 0 : masm.bind(&done);
8051 : }
8052 0 : }
8053 :
8054 : void
8055 0 : CodeGenerator::visitIsNullOrLikeUndefinedAndBranchT(LIsNullOrLikeUndefinedAndBranchT* lir)
8056 : {
8057 0 : DebugOnly<MCompare::CompareType> compareType = lir->cmpMir()->compareType();
8058 0 : MOZ_ASSERT(compareType == MCompare::Compare_Undefined ||
8059 : compareType == MCompare::Compare_Null);
8060 :
8061 0 : MIRType lhsType = lir->cmpMir()->lhs()->type();
8062 0 : MOZ_ASSERT(lhsType == MIRType::Object || lhsType == MIRType::ObjectOrNull);
8063 :
8064 0 : JSOp op = lir->cmpMir()->jsop();
8065 0 : MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || op == JSOP_EQ || op == JSOP_NE,
8066 : "Strict equality should have been folded");
8067 :
8068 0 : MOZ_ASSERT(lhsType == MIRType::ObjectOrNull || lir->cmpMir()->operandMightEmulateUndefined(),
8069 : "If the object couldn't emulate undefined, this should have been folded.");
8070 :
8071 : MBasicBlock* ifTrue;
8072 : MBasicBlock* ifFalse;
8073 :
8074 0 : if (op == JSOP_EQ || op == JSOP_STRICTEQ) {
8075 0 : ifTrue = lir->ifTrue();
8076 0 : ifFalse = lir->ifFalse();
8077 : } else {
8078 : // Swap branches.
8079 0 : ifTrue = lir->ifFalse();
8080 0 : ifFalse = lir->ifTrue();
8081 : }
8082 :
8083 0 : Register input = ToRegister(lir->getOperand(0));
8084 :
8085 0 : if ((op == JSOP_EQ || op == JSOP_NE) && lir->cmpMir()->operandMightEmulateUndefined()) {
8086 0 : OutOfLineTestObject* ool = new(alloc()) OutOfLineTestObject();
8087 0 : addOutOfLineCode(ool, lir->cmpMir());
8088 :
8089 0 : Label* ifTrueLabel = getJumpLabelForBranch(ifTrue);
8090 0 : Label* ifFalseLabel = getJumpLabelForBranch(ifFalse);
8091 :
8092 0 : if (lhsType == MIRType::ObjectOrNull)
8093 0 : masm.branchTestPtr(Assembler::Zero, input, input, ifTrueLabel);
8094 :
8095 : // Objects that emulate undefined are loosely equal to null/undefined.
8096 0 : Register scratch = ToRegister(lir->temp());
8097 0 : testObjectEmulatesUndefined(input, ifTrueLabel, ifFalseLabel, scratch, ool);
8098 : } else {
8099 0 : MOZ_ASSERT(lhsType == MIRType::ObjectOrNull);
8100 0 : testZeroEmitBranch(Assembler::Equal, input, ifTrue, ifFalse);
8101 : }
8102 0 : }
8103 :
8104 : void
8105 0 : CodeGenerator::emitSameValue(FloatRegister left, FloatRegister right, FloatRegister temp,
8106 : Register output)
8107 : {
8108 0 : Label nonEqual, isSameValue, isNotSameValue;
8109 0 : masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, left, right, &nonEqual);
8110 : {
8111 : // First, test for being equal to 0.0, which also includes -0.0.
8112 0 : masm.loadConstantDouble(0.0, temp);
8113 0 : masm.branchDouble(Assembler::DoubleNotEqual, left, temp, &isSameValue);
8114 :
8115 : // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0
8116 : // is -Infinity instead of Infinity.
8117 0 : Label isNegInf;
8118 0 : masm.loadConstantDouble(1.0, temp);
8119 0 : masm.divDouble(left, temp);
8120 0 : masm.branchDouble(Assembler::DoubleLessThan, temp, left, &isNegInf);
8121 : {
8122 0 : masm.loadConstantDouble(1.0, temp);
8123 0 : masm.divDouble(right, temp);
8124 0 : masm.branchDouble(Assembler::DoubleGreaterThan, temp, right, &isSameValue);
8125 0 : masm.jump(&isNotSameValue);
8126 : }
8127 0 : masm.bind(&isNegInf);
8128 : {
8129 0 : masm.loadConstantDouble(1.0, temp);
8130 0 : masm.divDouble(right, temp);
8131 0 : masm.branchDouble(Assembler::DoubleLessThan, temp, right, &isSameValue);
8132 0 : masm.jump(&isNotSameValue);
8133 : }
8134 : }
8135 0 : masm.bind(&nonEqual);
8136 : {
8137 : // Test if both values are NaN.
8138 0 : masm.branchDouble(Assembler::DoubleOrdered, left, left, &isNotSameValue);
8139 0 : masm.branchDouble(Assembler::DoubleOrdered, right, right, &isNotSameValue);
8140 : }
8141 :
8142 0 : Label done;
8143 0 : masm.bind(&isSameValue);
8144 0 : masm.move32(Imm32(1), output);
8145 0 : masm.jump(&done);
8146 :
8147 0 : masm.bind(&isNotSameValue);
8148 0 : masm.move32(Imm32(0), output);
8149 :
8150 0 : masm.bind(&done);
8151 0 : }
8152 :
8153 : void
8154 0 : CodeGenerator::visitSameValueD(LSameValueD* lir)
8155 : {
8156 0 : FloatRegister left = ToFloatRegister(lir->left());
8157 0 : FloatRegister right = ToFloatRegister(lir->right());
8158 0 : FloatRegister temp = ToFloatRegister(lir->tempFloat());
8159 0 : Register output = ToRegister(lir->output());
8160 :
8161 0 : emitSameValue(left, right, temp, output);
8162 0 : }
8163 :
8164 : void
8165 0 : CodeGenerator::visitSameValueV(LSameValueV* lir)
8166 : {
8167 0 : ValueOperand left = ToValue(lir, LSameValueV::LhsInput);
8168 0 : FloatRegister right = ToFloatRegister(lir->right());
8169 0 : FloatRegister temp1 = ToFloatRegister(lir->tempFloat1());
8170 0 : FloatRegister temp2 = ToFloatRegister(lir->tempFloat2());
8171 0 : Register output = ToRegister(lir->output());
8172 :
8173 0 : Label nonDouble;
8174 0 : masm.move32(Imm32(0), output);
8175 0 : masm.ensureDouble(left, temp1, &nonDouble);
8176 0 : emitSameValue(temp1, right, temp2, output);
8177 0 : masm.bind(&nonDouble);
8178 0 : }
8179 :
8180 : typedef bool (*SameValueFn)(JSContext*, HandleValue, HandleValue, bool*);
8181 0 : static const VMFunction SameValueInfo = FunctionInfo<SameValueFn>(js::SameValue, "SameValue");
8182 :
8183 : void
8184 0 : CodeGenerator::visitSameValueVM(LSameValueVM* lir)
8185 : {
8186 0 : pushArg(ToValue(lir, LSameValueVM::RhsInput));
8187 0 : pushArg(ToValue(lir, LSameValueVM::LhsInput));
8188 0 : callVM(SameValueInfo, lir);
8189 0 : }
8190 :
8191 : typedef JSString* (*ConcatStringsFn)(JSContext*, HandleString, HandleString);
8192 0 : static const VMFunction ConcatStringsInfo =
8193 0 : FunctionInfo<ConcatStringsFn>(ConcatStrings<CanGC>, "ConcatStrings");
8194 :
8195 : void
8196 48 : CodeGenerator::emitConcat(LInstruction* lir, Register lhs, Register rhs, Register output)
8197 : {
8198 0 : OutOfLineCode* ool = oolCallVM(ConcatStringsInfo, lir, ArgList(lhs, rhs),
8199 144 : StoreRegisterTo(output));
8200 :
8201 0 : const JitRealm* jitRealm = gen->realm->jitRealm();
8202 96 : JitCode* stringConcatStub = jitRealm->stringConcatStubNoBarrier(&realmStubsToReadBarrier_);
8203 0 : masm.call(stringConcatStub);
8204 0 : masm.branchTestPtr(Assembler::Zero, output, output, ool->entry());
8205 :
8206 0 : masm.bind(ool->rejoin());
8207 0 : }
8208 :
8209 : void
8210 48 : CodeGenerator::visitConcat(LConcat* lir)
8211 : {
8212 0 : Register lhs = ToRegister(lir->lhs());
8213 96 : Register rhs = ToRegister(lir->rhs());
8214 :
8215 0 : Register output = ToRegister(lir->output());
8216 :
8217 0 : MOZ_ASSERT(lhs == CallTempReg0);
8218 0 : MOZ_ASSERT(rhs == CallTempReg1);
8219 96 : MOZ_ASSERT(ToRegister(lir->temp1()) == CallTempReg0);
8220 0 : MOZ_ASSERT(ToRegister(lir->temp2()) == CallTempReg1);
8221 96 : MOZ_ASSERT(ToRegister(lir->temp3()) == CallTempReg2);
8222 0 : MOZ_ASSERT(ToRegister(lir->temp4()) == CallTempReg3);
8223 0 : MOZ_ASSERT(ToRegister(lir->temp5()) == CallTempReg4);
8224 0 : MOZ_ASSERT(output == CallTempReg5);
8225 :
8226 0 : emitConcat(lir, lhs, rhs, output);
8227 0 : }
8228 :
8229 : static void
8230 14 : CopyStringChars(MacroAssembler& masm, Register to, Register from, Register len,
8231 : Register byteOpScratch, size_t fromWidth, size_t toWidth)
8232 : {
8233 : // Copy |len| char16_t code units from |from| to |to|. Assumes len > 0
8234 : // (checked below in debug builds), and when done |to| must point to the
8235 : // next available char.
8236 :
8237 : #ifdef DEBUG
8238 28 : Label ok;
8239 28 : masm.branch32(Assembler::GreaterThan, len, Imm32(0), &ok);
8240 14 : masm.assumeUnreachable("Length should be greater than 0.");
8241 14 : masm.bind(&ok);
8242 : #endif
8243 :
8244 0 : MOZ_ASSERT(fromWidth == 1 || fromWidth == 2);
8245 0 : MOZ_ASSERT(toWidth == 1 || toWidth == 2);
8246 0 : MOZ_ASSERT_IF(toWidth == 1, fromWidth == 1);
8247 :
8248 28 : Label start;
8249 0 : masm.bind(&start);
8250 0 : if (fromWidth == 2)
8251 0 : masm.load16ZeroExtend(Address(from, 0), byteOpScratch);
8252 : else
8253 0 : masm.load8ZeroExtend(Address(from, 0), byteOpScratch);
8254 0 : if (toWidth == 2)
8255 0 : masm.store16(byteOpScratch, Address(to, 0));
8256 : else
8257 6 : masm.store8(byteOpScratch, Address(to, 0));
8258 0 : masm.addPtr(Imm32(fromWidth), from);
8259 0 : masm.addPtr(Imm32(toWidth), to);
8260 0 : masm.branchSub32(Assembler::NonZero, Imm32(1), len, &start);
8261 14 : }
8262 :
8263 : static void
8264 0 : CopyStringCharsMaybeInflate(MacroAssembler& masm, Register input, Register destChars,
8265 : Register temp1, Register temp2)
8266 : {
8267 : // destChars is TwoByte and input is a Latin1 or TwoByte string, so we may
8268 : // have to inflate.
8269 :
8270 6 : Label isLatin1, done;
8271 2 : masm.loadStringLength(input, temp1);
8272 2 : masm.branchLatin1String(input, &isLatin1);
8273 : {
8274 2 : masm.loadStringChars(input, temp2, CharEncoding::TwoByte);
8275 0 : masm.movePtr(temp2, input);
8276 0 : CopyStringChars(masm, destChars, input, temp1, temp2, sizeof(char16_t), sizeof(char16_t));
8277 0 : masm.jump(&done);
8278 : }
8279 0 : masm.bind(&isLatin1);
8280 : {
8281 0 : masm.loadStringChars(input, temp2, CharEncoding::Latin1);
8282 0 : masm.movePtr(temp2, input);
8283 2 : CopyStringChars(masm, destChars, input, temp1, temp2, sizeof(char), sizeof(char16_t));
8284 : }
8285 2 : masm.bind(&done);
8286 0 : }
8287 :
8288 : static void
8289 2 : ConcatInlineString(MacroAssembler& masm, Register lhs, Register rhs, Register output,
8290 : Register temp1, Register temp2, Register temp3,
8291 : bool stringsCanBeInNursery,
8292 : Label* failure, bool isTwoByte)
8293 : {
8294 : // State: result length in temp2.
8295 :
8296 : // Ensure both strings are linear.
8297 2 : masm.branchIfRope(lhs, failure);
8298 2 : masm.branchIfRope(rhs, failure);
8299 :
8300 : // Allocate a JSThinInlineString or JSFatInlineString.
8301 : size_t maxThinInlineLength;
8302 0 : if (isTwoByte)
8303 : maxThinInlineLength = JSThinInlineString::MAX_LENGTH_TWO_BYTE;
8304 : else
8305 1 : maxThinInlineLength = JSThinInlineString::MAX_LENGTH_LATIN1;
8306 :
8307 0 : Label isFat, allocDone;
8308 4 : masm.branch32(Assembler::Above, temp2, Imm32(maxThinInlineLength), &isFat);
8309 : {
8310 0 : uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
8311 2 : if (!isTwoByte)
8312 0 : flags |= JSString::LATIN1_CHARS_BIT;
8313 0 : masm.newGCString(output, temp1, failure, stringsCanBeInNursery);
8314 6 : masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
8315 0 : masm.jump(&allocDone);
8316 : }
8317 0 : masm.bind(&isFat);
8318 : {
8319 0 : uint32_t flags = JSString::INIT_FAT_INLINE_FLAGS;
8320 0 : if (!isTwoByte)
8321 1 : flags |= JSString::LATIN1_CHARS_BIT;
8322 0 : masm.newGCFatInlineString(output, temp1, failure, stringsCanBeInNursery);
8323 6 : masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
8324 : }
8325 0 : masm.bind(&allocDone);
8326 :
8327 : // Store length.
8328 0 : masm.store32(temp2, Address(output, JSString::offsetOfLength()));
8329 :
8330 : // Load chars pointer in temp2.
8331 2 : masm.loadInlineStringCharsForStore(output, temp2);
8332 :
8333 : {
8334 : // Copy lhs chars. Note that this advances temp2 to point to the next
8335 : // char. This also clobbers the lhs register.
8336 0 : if (isTwoByte) {
8337 1 : CopyStringCharsMaybeInflate(masm, lhs, temp2, temp1, temp3);
8338 : } else {
8339 1 : masm.loadStringLength(lhs, temp3);
8340 1 : masm.loadStringChars(lhs, temp1, CharEncoding::Latin1);
8341 0 : masm.movePtr(temp1, lhs);
8342 0 : CopyStringChars(masm, temp2, lhs, temp3, temp1, sizeof(char), sizeof(char));
8343 : }
8344 :
8345 : // Copy rhs chars. Clobbers the rhs register.
8346 0 : if (isTwoByte) {
8347 0 : CopyStringCharsMaybeInflate(masm, rhs, temp2, temp1, temp3);
8348 : } else {
8349 1 : masm.loadStringLength(rhs, temp3);
8350 1 : masm.loadStringChars(rhs, temp1, CharEncoding::Latin1);
8351 0 : masm.movePtr(temp1, rhs);
8352 0 : CopyStringChars(masm, temp2, rhs, temp3, temp1, sizeof(char), sizeof(char));
8353 : }
8354 :
8355 : // Null-terminate.
8356 0 : if (isTwoByte)
8357 0 : masm.store16(Imm32(0), Address(temp2, 0));
8358 : else
8359 2 : masm.store8(Imm32(0), Address(temp2, 0));
8360 : }
8361 :
8362 0 : masm.ret();
8363 2 : }
8364 :
8365 : typedef JSString* (*SubstringKernelFn)(JSContext* cx, HandleString str, int32_t begin, int32_t len);
8366 1 : static const VMFunction SubstringKernelInfo =
8367 0 : FunctionInfo<SubstringKernelFn>(SubstringKernel, "SubstringKernel");
8368 :
8369 : void
8370 3 : CodeGenerator::visitSubstr(LSubstr* lir)
8371 : {
8372 0 : Register string = ToRegister(lir->string());
8373 6 : Register begin = ToRegister(lir->begin());
8374 6 : Register length = ToRegister(lir->length());
8375 0 : Register output = ToRegister(lir->output());
8376 6 : Register temp = ToRegister(lir->temp());
8377 0 : Register temp3 = ToRegister(lir->temp3());
8378 :
8379 : // On x86 there are not enough registers. In that case reuse the string
8380 : // register as temporary.
8381 0 : Register temp2 = lir->temp2()->isBogusTemp() ? string : ToRegister(lir->temp2());
8382 :
8383 6 : Address stringFlags(string, JSString::offsetOfFlags());
8384 :
8385 15 : Label isLatin1, notInline, nonZero, isInlinedLatin1;
8386 :
8387 : // For every edge case use the C++ variant.
8388 : // Note: we also use this upon allocation failure in newGCString and
8389 : // newGCFatInlineString. To squeeze out even more performance those failures
8390 : // can be handled by allocate in ool code and returning to jit code to fill
8391 : // in all data.
8392 3 : OutOfLineCode* ool = oolCallVM(SubstringKernelInfo, lir,
8393 6 : ArgList(string, begin, length),
8394 9 : StoreRegisterTo(output));
8395 3 : Label* slowPath = ool->entry();
8396 3 : Label* done = ool->rejoin();
8397 :
8398 : // Zero length, return emptystring.
8399 0 : masm.branchTest32(Assembler::NonZero, length, length, &nonZero);
8400 0 : const JSAtomState& names = gen->runtime->names();
8401 0 : masm.movePtr(ImmGCPtr(names.empty), output);
8402 6 : masm.jump(done);
8403 :
8404 : // Use slow path for ropes.
8405 0 : masm.bind(&nonZero);
8406 0 : masm.branchIfRopeOrExternal(string, temp, slowPath);
8407 :
8408 : // Handle inlined strings by creating a FatInlineString.
8409 3 : masm.branchTest32(Assembler::Zero, stringFlags, Imm32(JSString::INLINE_CHARS_BIT), ¬Inline);
8410 0 : masm.newGCFatInlineString(output, temp, slowPath, stringsCanBeInNursery());
8411 0 : masm.store32(length, Address(output, JSString::offsetOfLength()));
8412 :
8413 6 : masm.branchLatin1String(string, &isInlinedLatin1);
8414 : {
8415 0 : masm.store32(Imm32(JSString::INIT_FAT_INLINE_FLAGS),
8416 0 : Address(output, JSString::offsetOfFlags()));
8417 3 : masm.loadInlineStringChars(string, temp, CharEncoding::TwoByte);
8418 0 : if (temp2 == string)
8419 0 : masm.push(string);
8420 0 : BaseIndex chars(temp, begin, ScaleFromElemWidth(sizeof(char16_t)));
8421 0 : masm.computeEffectiveAddress(chars, temp2);
8422 0 : masm.loadInlineStringCharsForStore(output, temp);
8423 0 : CopyStringChars(masm, temp, temp2, length, temp3, sizeof(char16_t), sizeof(char16_t));
8424 1 : masm.load32(Address(output, JSString::offsetOfLength()), length);
8425 0 : masm.store16(Imm32(0), Address(temp, 0));
8426 0 : if (temp2 == string)
8427 0 : masm.pop(string);
8428 0 : masm.jump(done);
8429 : }
8430 0 : masm.bind(&isInlinedLatin1);
8431 : {
8432 1 : masm.store32(Imm32(JSString::INIT_FAT_INLINE_FLAGS | JSString::LATIN1_CHARS_BIT),
8433 0 : Address(output, JSString::offsetOfFlags()));
8434 3 : if (temp2 == string) {
8435 0 : masm.push(string);
8436 0 : masm.loadInlineStringChars(string, temp, CharEncoding::Latin1);
8437 0 : masm.movePtr(temp, temp2);
8438 : } else {
8439 0 : masm.loadInlineStringChars(string, temp2, CharEncoding::Latin1);
8440 : }
8441 : static_assert(sizeof(char) == 1, "begin index shouldn't need scaling");
8442 1 : masm.addPtr(begin, temp2);
8443 3 : masm.loadInlineStringCharsForStore(output, temp);
8444 0 : CopyStringChars(masm, temp, temp2, length, temp3, sizeof(char), sizeof(char));
8445 6 : masm.load32(Address(output, JSString::offsetOfLength()), length);
8446 6 : masm.store8(Imm32(0), Address(temp, 0));
8447 0 : if (temp2 == string)
8448 0 : masm.pop(string);
8449 0 : masm.jump(done);
8450 : }
8451 :
8452 : // Handle other cases with a DependentString.
8453 1 : masm.bind(¬Inline);
8454 0 : masm.newGCString(output, temp, slowPath, gen->stringsCanBeInNursery());
8455 6 : masm.store32(length, Address(output, JSString::offsetOfLength()));
8456 3 : masm.storeDependentStringBase(string, output);
8457 :
8458 0 : masm.branchLatin1String(string, &isLatin1);
8459 : {
8460 0 : masm.store32(Imm32(JSString::DEPENDENT_FLAGS), Address(output, JSString::offsetOfFlags()));
8461 0 : masm.loadNonInlineStringChars(string, temp, CharEncoding::TwoByte);
8462 6 : BaseIndex chars(temp, begin, ScaleFromElemWidth(sizeof(char16_t)));
8463 0 : masm.computeEffectiveAddress(chars, temp);
8464 3 : masm.storeNonInlineStringChars(temp, output);
8465 0 : masm.jump(done);
8466 : }
8467 0 : masm.bind(&isLatin1);
8468 : {
8469 0 : masm.store32(Imm32(JSString::DEPENDENT_FLAGS | JSString::LATIN1_CHARS_BIT),
8470 0 : Address(output, JSString::offsetOfFlags()));
8471 3 : masm.loadNonInlineStringChars(string, temp, CharEncoding::Latin1);
8472 : static_assert(sizeof(char) == 1, "begin index shouldn't need scaling");
8473 6 : masm.addPtr(begin, temp);
8474 0 : masm.storeNonInlineStringChars(temp, output);
8475 0 : masm.jump(done);
8476 : }
8477 :
8478 0 : masm.bind(done);
8479 0 : }
8480 :
8481 : JitCode*
8482 1 : JitRealm::generateStringConcatStub(JSContext* cx)
8483 : {
8484 0 : StackMacroAssembler masm(cx);
8485 :
8486 1 : Register lhs = CallTempReg0;
8487 0 : Register rhs = CallTempReg1;
8488 1 : Register temp1 = CallTempReg2;
8489 0 : Register temp2 = CallTempReg3;
8490 1 : Register temp3 = CallTempReg4;
8491 0 : Register output = CallTempReg5;
8492 :
8493 0 : Label failure;
8494 : #ifdef JS_USE_LINK_REGISTER
8495 : masm.pushReturnAddress();
8496 : #endif
8497 : // If lhs is empty, return rhs.
8498 0 : Label leftEmpty;
8499 1 : masm.loadStringLength(lhs, temp1);
8500 1 : masm.branchTest32(Assembler::Zero, temp1, temp1, &leftEmpty);
8501 :
8502 : // If rhs is empty, return lhs.
8503 0 : Label rightEmpty;
8504 0 : masm.loadStringLength(rhs, temp2);
8505 0 : masm.branchTest32(Assembler::Zero, temp2, temp2, &rightEmpty);
8506 :
8507 1 : masm.add32(temp1, temp2);
8508 :
8509 : // Check if we can use a JSFatInlineString. The result is a Latin1 string if
8510 : // lhs and rhs are both Latin1, so we AND the flags.
8511 3 : Label isFatInlineTwoByte, isFatInlineLatin1;
8512 0 : masm.load32(Address(lhs, JSString::offsetOfFlags()), temp1);
8513 2 : masm.and32(Address(rhs, JSString::offsetOfFlags()), temp1);
8514 :
8515 3 : Label isLatin1, notInline;
8516 0 : masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSString::LATIN1_CHARS_BIT), &isLatin1);
8517 : {
8518 0 : masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE),
8519 1 : &isFatInlineTwoByte);
8520 0 : masm.jump(¬Inline);
8521 : }
8522 1 : masm.bind(&isLatin1);
8523 : {
8524 0 : masm.branch32(Assembler::BelowOrEqual, temp2, Imm32(JSFatInlineString::MAX_LENGTH_LATIN1),
8525 0 : &isFatInlineLatin1);
8526 : }
8527 0 : masm.bind(¬Inline);
8528 :
8529 : // Keep AND'ed flags in temp1.
8530 :
8531 : // Ensure result length <= JSString::MAX_LENGTH.
8532 0 : masm.branch32(Assembler::Above, temp2, Imm32(JSString::MAX_LENGTH), &failure);
8533 :
8534 : // Allocate a new rope, guaranteed to be in the nursery.
8535 1 : masm.newGCString(output, temp3, &failure, stringsCanBeInNursery);
8536 :
8537 : // Store rope length and flags. temp1 still holds the result of AND'ing the
8538 : // lhs and rhs flags, so we just have to clear the other flags and set
8539 : // NON_ATOM_BIT to get our rope flags (Latin1 if both lhs and rhs are
8540 : // Latin1).
8541 : static_assert(JSString::INIT_ROPE_FLAGS == JSString::NON_ATOM_BIT,
8542 : "Rope type flags must be NON_ATOM_BIT only");
8543 1 : masm.and32(Imm32(JSString::LATIN1_CHARS_BIT), temp1);
8544 1 : masm.or32(Imm32(JSString::NON_ATOM_BIT), temp1);
8545 2 : masm.store32(temp1, Address(output, JSString::offsetOfFlags()));
8546 2 : masm.store32(temp2, Address(output, JSString::offsetOfLength()));
8547 :
8548 : // Store left and right nodes.
8549 0 : masm.storeRopeChildren(lhs, rhs, output);
8550 0 : masm.ret();
8551 :
8552 1 : masm.bind(&leftEmpty);
8553 1 : masm.mov(rhs, output);
8554 0 : masm.ret();
8555 :
8556 1 : masm.bind(&rightEmpty);
8557 0 : masm.mov(lhs, output);
8558 0 : masm.ret();
8559 :
8560 1 : masm.bind(&isFatInlineTwoByte);
8561 0 : ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
8562 0 : stringsCanBeInNursery, &failure, true);
8563 :
8564 1 : masm.bind(&isFatInlineLatin1);
8565 0 : ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
8566 0 : stringsCanBeInNursery, &failure, false);
8567 :
8568 1 : masm.pop(temp2);
8569 0 : masm.pop(temp1);
8570 :
8571 0 : masm.bind(&failure);
8572 2 : masm.movePtr(ImmPtr(nullptr), output);
8573 0 : masm.ret();
8574 :
8575 2 : Linker linker(masm);
8576 0 : AutoFlushICache afc("StringConcatStub");
8577 0 : JitCode* code = linker.newCode(cx, CodeKind::Other);
8578 :
8579 : #ifdef JS_ION_PERF
8580 : writePerfSpewerJitCodeProfile(code, "StringConcatStub");
8581 : #endif
8582 : #ifdef MOZ_VTUNE
8583 : vtune::MarkStub(code, "StringConcatStub");
8584 : #endif
8585 :
8586 1 : return code;
8587 : }
8588 :
8589 : void
8590 4 : JitRuntime::generateMallocStub(MacroAssembler& masm)
8591 : {
8592 4 : const Register regReturn = CallTempReg0;
8593 4 : const Register regZone = CallTempReg0;
8594 4 : const Register regNBytes = CallTempReg1;
8595 :
8596 8 : mallocStubOffset_ = startTrampolineCode(masm);
8597 :
8598 0 : AllocatableRegisterSet regs(RegisterSet::Volatile());
8599 : #ifdef JS_USE_LINK_REGISTER
8600 : masm.pushReturnAddress();
8601 : #endif
8602 4 : regs.takeUnchecked(regZone);
8603 0 : regs.takeUnchecked(regNBytes);
8604 4 : LiveRegisterSet save(regs.asLiveSet());
8605 4 : masm.PushRegsInMask(save);
8606 :
8607 0 : const Register regTemp = regs.takeAnyGeneral();
8608 0 : MOZ_ASSERT(regTemp != regNBytes && regTemp != regZone);
8609 :
8610 0 : masm.setupUnalignedABICall(regTemp);
8611 4 : masm.passABIArg(regZone);
8612 0 : masm.passABIArg(regNBytes);
8613 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, MallocWrapper));
8614 4 : masm.storeCallPointerResult(regReturn);
8615 :
8616 0 : masm.PopRegsInMask(save);
8617 0 : masm.ret();
8618 0 : }
8619 :
8620 : void
8621 0 : JitRuntime::generateFreeStub(MacroAssembler& masm)
8622 : {
8623 0 : const Register regSlots = CallTempReg0;
8624 :
8625 8 : freeStubOffset_ = startTrampolineCode(masm);
8626 :
8627 : #ifdef JS_USE_LINK_REGISTER
8628 : masm.pushReturnAddress();
8629 : #endif
8630 0 : AllocatableRegisterSet regs(RegisterSet::Volatile());
8631 4 : regs.takeUnchecked(regSlots);
8632 4 : LiveRegisterSet save(regs.asLiveSet());
8633 4 : masm.PushRegsInMask(save);
8634 :
8635 0 : const Register regTemp = regs.takeAnyGeneral();
8636 0 : MOZ_ASSERT(regTemp != regSlots);
8637 :
8638 0 : masm.setupUnalignedABICall(regTemp);
8639 4 : masm.passABIArg(regSlots);
8640 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js_free), MoveOp::GENERAL,
8641 0 : CheckUnsafeCallWithABI::DontCheckOther);
8642 :
8643 0 : masm.PopRegsInMask(save);
8644 :
8645 0 : masm.ret();
8646 0 : }
8647 :
8648 : void
8649 4 : JitRuntime::generateLazyLinkStub(MacroAssembler& masm)
8650 : {
8651 0 : lazyLinkStubOffset_ = startTrampolineCode(masm);
8652 :
8653 : #ifdef JS_USE_LINK_REGISTER
8654 : masm.pushReturnAddress();
8655 : #endif
8656 :
8657 8 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
8658 4 : Register temp0 = regs.takeAny();
8659 4 : Register temp1 = regs.takeAny();
8660 4 : Register temp2 = regs.takeAny();
8661 :
8662 0 : masm.loadJSContext(temp0);
8663 0 : masm.enterFakeExitFrame(temp0, temp2, ExitFrameType::LazyLink);
8664 0 : masm.moveStackPtrTo(temp1);
8665 :
8666 4 : masm.setupUnalignedABICall(temp2);
8667 0 : masm.passABIArg(temp0);
8668 0 : masm.passABIArg(temp1);
8669 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, LazyLinkTopActivation), MoveOp::GENERAL,
8670 4 : CheckUnsafeCallWithABI::DontCheckHasExitFrame);
8671 :
8672 0 : masm.leaveExitFrame();
8673 :
8674 : #ifdef JS_USE_LINK_REGISTER
8675 : // Restore the return address such that the emitPrologue function of the
8676 : // CodeGenerator can push it back on the stack with pushReturnAddress.
8677 : masm.popReturnAddress();
8678 : #endif
8679 4 : masm.jump(ReturnReg);
8680 4 : }
8681 :
8682 : void
8683 4 : JitRuntime::generateInterpreterStub(MacroAssembler& masm)
8684 : {
8685 0 : interpreterStubOffset_ = startTrampolineCode(masm);
8686 :
8687 : #ifdef JS_USE_LINK_REGISTER
8688 : masm.pushReturnAddress();
8689 : #endif
8690 :
8691 8 : AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
8692 4 : Register temp0 = regs.takeAny();
8693 4 : Register temp1 = regs.takeAny();
8694 4 : Register temp2 = regs.takeAny();
8695 :
8696 0 : masm.loadJSContext(temp0);
8697 0 : masm.enterFakeExitFrame(temp0, temp2, ExitFrameType::InterpreterStub);
8698 0 : masm.moveStackPtrTo(temp1);
8699 :
8700 4 : masm.setupUnalignedABICall(temp2);
8701 0 : masm.passABIArg(temp0);
8702 0 : masm.passABIArg(temp1);
8703 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, InvokeFromInterpreterStub), MoveOp::GENERAL,
8704 4 : CheckUnsafeCallWithABI::DontCheckHasExitFrame);
8705 :
8706 0 : masm.branchIfFalseBool(ReturnReg, masm.failureLabel());
8707 0 : masm.leaveExitFrame();
8708 :
8709 : // InvokeFromInterpreterStub stores the return value in argv[0], where the
8710 : // caller stored |this|.
8711 0 : masm.loadValue(Address(masm.getStackPointer(), JitFrameLayout::offsetOfThis()),
8712 0 : JSReturnOperand);
8713 8 : masm.ret();
8714 4 : }
8715 :
8716 : bool
8717 0 : JitRuntime::generateTLEventVM(MacroAssembler& masm, const VMFunction& f, bool enter)
8718 : {
8719 : #ifdef JS_TRACE_LOGGING
8720 1816 : bool vmEventEnabled = TraceLogTextIdEnabled(TraceLogger_VM);
8721 1816 : bool vmSpecificEventEnabled = TraceLogTextIdEnabled(TraceLogger_VMSpecific);
8722 :
8723 1816 : if (vmEventEnabled || vmSpecificEventEnabled) {
8724 0 : AllocatableRegisterSet regs(RegisterSet::Volatile());
8725 0 : Register loggerReg = regs.takeAnyGeneral();
8726 0 : masm.Push(loggerReg);
8727 0 : masm.loadTraceLogger(loggerReg);
8728 :
8729 0 : if (vmEventEnabled) {
8730 0 : if (enter)
8731 0 : masm.tracelogStartId(loggerReg, TraceLogger_VM, /* force = */ true);
8732 : else
8733 0 : masm.tracelogStopId(loggerReg, TraceLogger_VM, /* force = */ true);
8734 : }
8735 0 : if (vmSpecificEventEnabled) {
8736 0 : TraceLoggerEvent event(f.name());
8737 0 : if (!event.hasTextId())
8738 0 : return false;
8739 :
8740 0 : if (enter)
8741 0 : masm.tracelogStartId(loggerReg, event.textId(), /* force = */ true);
8742 : else
8743 0 : masm.tracelogStopId(loggerReg, event.textId(), /* force = */ true);
8744 : }
8745 :
8746 0 : masm.Pop(loggerReg);
8747 : }
8748 : #endif
8749 :
8750 : return true;
8751 : }
8752 :
8753 : typedef bool (*CharCodeAtFn)(JSContext*, HandleString, int32_t, uint32_t*);
8754 1 : static const VMFunction CharCodeAtInfo =
8755 3 : FunctionInfo<CharCodeAtFn>(jit::CharCodeAt, "CharCodeAt");
8756 :
8757 : void
8758 0 : CodeGenerator::visitCharCodeAt(LCharCodeAt* lir)
8759 : {
8760 0 : Register str = ToRegister(lir->str());
8761 0 : Register index = ToRegister(lir->index());
8762 0 : Register output = ToRegister(lir->output());
8763 0 : Register temp = ToRegister(lir->temp());
8764 :
8765 0 : OutOfLineCode* ool = oolCallVM(CharCodeAtInfo, lir, ArgList(str, index), StoreRegisterTo(output));
8766 0 : masm.loadStringChar(str, index, output, temp, ool->entry());
8767 0 : masm.bind(ool->rejoin());
8768 0 : }
8769 :
8770 : typedef JSFlatString* (*StringFromCharCodeFn)(JSContext*, int32_t);
8771 0 : static const VMFunction StringFromCharCodeInfo =
8772 0 : FunctionInfo<StringFromCharCodeFn>(jit::StringFromCharCode, "StringFromCharCode");
8773 :
8774 : void
8775 0 : CodeGenerator::visitFromCharCode(LFromCharCode* lir)
8776 : {
8777 0 : Register code = ToRegister(lir->code());
8778 0 : Register output = ToRegister(lir->output());
8779 :
8780 0 : OutOfLineCode* ool = oolCallVM(StringFromCharCodeInfo, lir, ArgList(code), StoreRegisterTo(output));
8781 :
8782 : // OOL path if code >= UNIT_STATIC_LIMIT.
8783 0 : masm.boundsCheck32PowerOfTwo(code, StaticStrings::UNIT_STATIC_LIMIT, ool->entry());
8784 :
8785 0 : masm.movePtr(ImmPtr(&gen->runtime->staticStrings().unitStaticTable), output);
8786 0 : masm.loadPtr(BaseIndex(output, code, ScalePointer), output);
8787 :
8788 0 : masm.bind(ool->rejoin());
8789 0 : }
8790 :
8791 : typedef JSString* (*StringFromCodePointFn)(JSContext*, int32_t);
8792 1 : static const VMFunction StringFromCodePointInfo =
8793 0 : FunctionInfo<StringFromCodePointFn>(jit::StringFromCodePoint, "StringFromCodePoint");
8794 :
8795 : void
8796 0 : CodeGenerator::visitFromCodePoint(LFromCodePoint* lir)
8797 : {
8798 0 : Register codePoint = ToRegister(lir->codePoint());
8799 0 : Register output = ToRegister(lir->output());
8800 0 : Register temp1 = ToRegister(lir->temp1());
8801 0 : Register temp2 = ToRegister(lir->temp2());
8802 0 : LSnapshot* snapshot = lir->snapshot();
8803 :
8804 : // The OOL path is only taken when we can't allocate the inline string.
8805 0 : OutOfLineCode* ool = oolCallVM(StringFromCodePointInfo, lir, ArgList(codePoint),
8806 0 : StoreRegisterTo(output));
8807 :
8808 0 : Label isTwoByte;
8809 0 : Label* done = ool->rejoin();
8810 :
8811 : static_assert(StaticStrings::UNIT_STATIC_LIMIT -1 == JSString::MAX_LATIN1_CHAR,
8812 : "Latin-1 strings can be loaded from static strings");
8813 0 : masm.boundsCheck32PowerOfTwo(codePoint, StaticStrings::UNIT_STATIC_LIMIT, &isTwoByte);
8814 : {
8815 0 : masm.movePtr(ImmPtr(&gen->runtime->staticStrings().unitStaticTable), output);
8816 0 : masm.loadPtr(BaseIndex(output, codePoint, ScalePointer), output);
8817 0 : masm.jump(done);
8818 : }
8819 0 : masm.bind(&isTwoByte);
8820 : {
8821 : // Use a bailout if the input is not a valid code point, because
8822 : // MFromCodePoint is movable and it'd be observable when a moved
8823 : // fromCodePoint throws an exception before its actual call site.
8824 0 : bailoutCmp32(Assembler::Above, codePoint, Imm32(unicode::NonBMPMax), snapshot);
8825 :
8826 : // Allocate a JSThinInlineString.
8827 : {
8828 : static_assert(JSThinInlineString::MAX_LENGTH_TWO_BYTE >= 2,
8829 : "JSThinInlineString can hold a supplementary code point");
8830 :
8831 0 : uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
8832 0 : masm.newGCString(output, temp1, ool->entry(), gen->stringsCanBeInNursery());
8833 0 : masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
8834 : }
8835 :
8836 0 : Label isSupplementary;
8837 0 : masm.branch32(Assembler::AboveOrEqual, codePoint, Imm32(unicode::NonBMPMin),
8838 0 : &isSupplementary);
8839 : {
8840 : // Store length.
8841 0 : masm.store32(Imm32(1), Address(output, JSString::offsetOfLength()));
8842 :
8843 : // Load chars pointer in temp1.
8844 0 : masm.loadInlineStringCharsForStore(output, temp1);
8845 :
8846 0 : masm.store16(codePoint, Address(temp1, 0));
8847 :
8848 : // Null-terminate.
8849 0 : masm.store16(Imm32(0), Address(temp1, sizeof(char16_t)));
8850 :
8851 0 : masm.jump(done);
8852 : }
8853 0 : masm.bind(&isSupplementary);
8854 : {
8855 : // Store length.
8856 0 : masm.store32(Imm32(2), Address(output, JSString::offsetOfLength()));
8857 :
8858 : // Load chars pointer in temp1.
8859 0 : masm.loadInlineStringCharsForStore(output, temp1);
8860 :
8861 : // Inlined unicode::LeadSurrogate(uint32_t).
8862 0 : masm.move32(codePoint, temp2);
8863 0 : masm.rshift32(Imm32(10), temp2);
8864 0 : masm.add32(Imm32(unicode::LeadSurrogateMin - (unicode::NonBMPMin >> 10)), temp2);
8865 :
8866 0 : masm.store16(temp2, Address(temp1, 0));
8867 :
8868 : // Inlined unicode::TrailSurrogate(uint32_t).
8869 0 : masm.move32(codePoint, temp2);
8870 0 : masm.and32(Imm32(0x3FF), temp2);
8871 0 : masm.or32(Imm32(unicode::TrailSurrogateMin), temp2);
8872 :
8873 0 : masm.store16(temp2, Address(temp1, sizeof(char16_t)));
8874 :
8875 : // Null-terminate.
8876 0 : masm.store16(Imm32(0), Address(temp1, 2 * sizeof(char16_t)));
8877 : }
8878 : }
8879 :
8880 0 : masm.bind(done);
8881 0 : }
8882 :
8883 : typedef JSString* (*StringToLowerCaseFn)(JSContext*, HandleString);
8884 1 : static const VMFunction StringToLowerCaseInfo =
8885 0 : FunctionInfo<StringToLowerCaseFn>(js::StringToLowerCase, "StringToLowerCase");
8886 :
8887 : typedef JSString* (*StringToUpperCaseFn)(JSContext*, HandleString);
8888 1 : static const VMFunction StringToUpperCaseInfo =
8889 0 : FunctionInfo<StringToUpperCaseFn>(js::StringToUpperCase, "StringToUpperCase");
8890 :
8891 : void
8892 0 : CodeGenerator::visitStringConvertCase(LStringConvertCase* lir)
8893 : {
8894 0 : pushArg(ToRegister(lir->string()));
8895 :
8896 0 : if (lir->mir()->mode() == MStringConvertCase::LowerCase)
8897 0 : callVM(StringToLowerCaseInfo, lir);
8898 : else
8899 0 : callVM(StringToUpperCaseInfo, lir);
8900 0 : }
8901 :
8902 : void
8903 0 : CodeGenerator::visitSinCos(LSinCos *lir)
8904 : {
8905 0 : Register temp = ToRegister(lir->temp());
8906 0 : Register params = ToRegister(lir->temp2());
8907 0 : FloatRegister input = ToFloatRegister(lir->input());
8908 0 : FloatRegister outputSin = ToFloatRegister(lir->outputSin());
8909 0 : FloatRegister outputCos = ToFloatRegister(lir->outputCos());
8910 :
8911 0 : masm.reserveStack(sizeof(double) * 2);
8912 0 : masm.moveStackPtrTo(params);
8913 :
8914 0 : const MathCache* mathCache = lir->mir()->cache();
8915 :
8916 0 : masm.setupUnalignedABICall(temp);
8917 0 : if (mathCache) {
8918 0 : masm.movePtr(ImmPtr(mathCache), temp);
8919 0 : masm.passABIArg(temp);
8920 : }
8921 :
8922 : #define MAYBE_CACHED_(fcn) (mathCache ? (void*)fcn ## _impl : (void*)fcn ## _uncached)
8923 :
8924 0 : masm.passABIArg(input, MoveOp::DOUBLE);
8925 0 : masm.passABIArg(MoveOperand(params, sizeof(double), MoveOperand::EFFECTIVE_ADDRESS),
8926 0 : MoveOp::GENERAL);
8927 0 : masm.passABIArg(MoveOperand(params, 0, MoveOperand::EFFECTIVE_ADDRESS),
8928 0 : MoveOp::GENERAL);
8929 :
8930 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, MAYBE_CACHED_(js::math_sincos)));
8931 : #undef MAYBE_CACHED_
8932 :
8933 0 : masm.loadDouble(Address(masm.getStackPointer(), 0), outputCos);
8934 0 : masm.loadDouble(Address(masm.getStackPointer(), sizeof(double)), outputSin);
8935 0 : masm.freeStack(sizeof(double) * 2);
8936 0 : }
8937 :
8938 : typedef ArrayObject* (*StringSplitFn)(JSContext*, HandleObjectGroup, HandleString, HandleString, uint32_t);
8939 1 : static const VMFunction StringSplitInfo =
8940 1 : FunctionInfo<StringSplitFn>(js::str_split_string, "str_split_string");
8941 :
8942 : void
8943 0 : CodeGenerator::visitStringSplit(LStringSplit* lir)
8944 : {
8945 0 : pushArg(Imm32(INT32_MAX));
8946 0 : pushArg(ToRegister(lir->separator()));
8947 0 : pushArg(ToRegister(lir->string()));
8948 0 : pushArg(ImmGCPtr(lir->mir()->group()));
8949 :
8950 0 : callVM(StringSplitInfo, lir);
8951 0 : }
8952 :
8953 : void
8954 46 : CodeGenerator::visitInitializedLength(LInitializedLength* lir)
8955 : {
8956 0 : Address initLength(ToRegister(lir->elements()), ObjectElements::offsetOfInitializedLength());
8957 92 : masm.load32(initLength, ToRegister(lir->output()));
8958 46 : }
8959 :
8960 : void
8961 0 : CodeGenerator::visitSetInitializedLength(LSetInitializedLength* lir)
8962 : {
8963 0 : Address initLength(ToRegister(lir->elements()), ObjectElements::offsetOfInitializedLength());
8964 100 : SetLengthFromIndex(masm, lir->index(), initLength);
8965 100 : }
8966 :
8967 : void
8968 0 : CodeGenerator::visitNotO(LNotO* lir)
8969 : {
8970 0 : MOZ_ASSERT(lir->mir()->operandMightEmulateUndefined(),
8971 : "This should be constant-folded if the object can't emulate undefined.");
8972 :
8973 0 : OutOfLineTestObjectWithLabels* ool = new(alloc()) OutOfLineTestObjectWithLabels();
8974 0 : addOutOfLineCode(ool, lir->mir());
8975 :
8976 0 : Label* ifEmulatesUndefined = ool->label1();
8977 0 : Label* ifDoesntEmulateUndefined = ool->label2();
8978 :
8979 0 : Register objreg = ToRegister(lir->input());
8980 0 : Register output = ToRegister(lir->output());
8981 : branchTestObjectEmulatesUndefined(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
8982 0 : output, ool);
8983 : // fall through
8984 :
8985 0 : Label join;
8986 :
8987 0 : masm.move32(Imm32(0), output);
8988 0 : masm.jump(&join);
8989 :
8990 0 : masm.bind(ifEmulatesUndefined);
8991 0 : masm.move32(Imm32(1), output);
8992 :
8993 0 : masm.bind(&join);
8994 0 : }
8995 :
8996 : void
8997 0 : CodeGenerator::visitNotV(LNotV* lir)
8998 : {
8999 0 : Maybe<Label> ifTruthyLabel, ifFalsyLabel;
9000 : Label* ifTruthy;
9001 : Label* ifFalsy;
9002 :
9003 0 : OutOfLineTestObjectWithLabels* ool = nullptr;
9004 0 : MDefinition* operand = lir->mir()->input();
9005 : // Unfortunately, it's possible that someone (e.g. phi elimination) switched
9006 : // out our operand after we did cacheOperandMightEmulateUndefined. So we
9007 : // might think it can emulate undefined _and_ know that it can't be an
9008 : // object.
9009 0 : if (lir->mir()->operandMightEmulateUndefined() && operand->mightBeType(MIRType::Object)) {
9010 0 : ool = new(alloc()) OutOfLineTestObjectWithLabels();
9011 0 : addOutOfLineCode(ool, lir->mir());
9012 0 : ifTruthy = ool->label1();
9013 0 : ifFalsy = ool->label2();
9014 : } else {
9015 0 : ifTruthyLabel.emplace();
9016 0 : ifFalsyLabel.emplace();
9017 0 : ifTruthy = ifTruthyLabel.ptr();
9018 0 : ifFalsy = ifFalsyLabel.ptr();
9019 : }
9020 :
9021 0 : testValueTruthyKernel(ToValue(lir, LNotV::Input), lir->temp1(), lir->temp2(),
9022 : ToFloatRegister(lir->tempFloat()),
9023 0 : ifTruthy, ifFalsy, ool, operand);
9024 :
9025 0 : Label join;
9026 0 : Register output = ToRegister(lir->output());
9027 :
9028 : // Note that the testValueTruthyKernel call above may choose to fall through
9029 : // to ifTruthy instead of branching there.
9030 0 : masm.bind(ifTruthy);
9031 0 : masm.move32(Imm32(0), output);
9032 0 : masm.jump(&join);
9033 :
9034 0 : masm.bind(ifFalsy);
9035 0 : masm.move32(Imm32(1), output);
9036 :
9037 : // both branches meet here.
9038 0 : masm.bind(&join);
9039 0 : }
9040 :
9041 : void
9042 27 : CodeGenerator::visitBoundsCheck(LBoundsCheck* lir)
9043 : {
9044 0 : const LAllocation* index = lir->index();
9045 27 : const LAllocation* length = lir->length();
9046 27 : LSnapshot* snapshot = lir->snapshot();
9047 :
9048 27 : if (index->isConstant()) {
9049 : // Use uint32 so that the comparison is unsigned.
9050 0 : uint32_t idx = ToInt32(index);
9051 0 : if (length->isConstant()) {
9052 0 : uint32_t len = ToInt32(lir->length());
9053 0 : if (idx < len)
9054 : return;
9055 0 : bailout(snapshot);
9056 0 : return;
9057 : }
9058 :
9059 27 : if (length->isRegister())
9060 0 : bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), Imm32(idx), snapshot);
9061 : else
9062 0 : bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), Imm32(idx), snapshot);
9063 : return;
9064 : }
9065 :
9066 0 : Register indexReg = ToRegister(index);
9067 0 : if (length->isConstant())
9068 0 : bailoutCmp32(Assembler::AboveOrEqual, indexReg, Imm32(ToInt32(length)), snapshot);
9069 0 : else if (length->isRegister())
9070 0 : bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), indexReg, snapshot);
9071 : else
9072 0 : bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), indexReg, snapshot);
9073 : }
9074 :
9075 : void
9076 12 : CodeGenerator::visitBoundsCheckRange(LBoundsCheckRange* lir)
9077 : {
9078 24 : int32_t min = lir->mir()->minimum();
9079 24 : int32_t max = lir->mir()->maximum();
9080 12 : MOZ_ASSERT(max >= min);
9081 :
9082 12 : const LAllocation* length = lir->length();
9083 0 : LSnapshot* snapshot = lir->snapshot();
9084 0 : Register temp = ToRegister(lir->getTemp(0));
9085 0 : if (lir->index()->isConstant()) {
9086 : int32_t nmin, nmax;
9087 0 : int32_t index = ToInt32(lir->index());
9088 0 : if (SafeAdd(index, min, &nmin) && SafeAdd(index, max, &nmax) && nmin >= 0) {
9089 0 : if (length->isRegister())
9090 0 : bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), Imm32(nmax), snapshot);
9091 : else
9092 0 : bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), Imm32(nmax), snapshot);
9093 : return;
9094 : }
9095 0 : masm.mov(ImmWord(index), temp);
9096 : } else {
9097 0 : masm.mov(ToRegister(lir->index()), temp);
9098 : }
9099 :
9100 : // If the minimum and maximum differ then do an underflow check first.
9101 : // If the two are the same then doing an unsigned comparison on the
9102 : // length will also catch a negative index.
9103 0 : if (min != max) {
9104 0 : if (min != 0) {
9105 0 : Label bail;
9106 0 : masm.branchAdd32(Assembler::Overflow, Imm32(min), temp, &bail);
9107 0 : bailoutFrom(&bail, snapshot);
9108 : }
9109 :
9110 0 : bailoutCmp32(Assembler::LessThan, temp, Imm32(0), snapshot);
9111 :
9112 0 : if (min != 0) {
9113 : int32_t diff;
9114 0 : if (SafeSub(max, min, &diff))
9115 : max = diff;
9116 : else
9117 0 : masm.sub32(Imm32(min), temp);
9118 : }
9119 : }
9120 :
9121 : // Compute the maximum possible index. No overflow check is needed when
9122 : // max > 0. We can only wraparound to a negative number, which will test as
9123 : // larger than all nonnegative numbers in the unsigned comparison, and the
9124 : // length is required to be nonnegative (else testing a negative length
9125 : // would succeed on any nonnegative index).
9126 0 : if (max != 0) {
9127 0 : if (max < 0) {
9128 0 : Label bail;
9129 0 : masm.branchAdd32(Assembler::Overflow, Imm32(max), temp, &bail);
9130 0 : bailoutFrom(&bail, snapshot);
9131 : } else {
9132 0 : masm.add32(Imm32(max), temp);
9133 : }
9134 : }
9135 :
9136 0 : if (length->isRegister())
9137 0 : bailoutCmp32(Assembler::BelowOrEqual, ToRegister(length), temp, snapshot);
9138 : else
9139 0 : bailoutCmp32(Assembler::BelowOrEqual, ToAddress(length), temp, snapshot);
9140 : }
9141 :
9142 : void
9143 0 : CodeGenerator::visitBoundsCheckLower(LBoundsCheckLower* lir)
9144 : {
9145 0 : int32_t min = lir->mir()->minimum();
9146 0 : bailoutCmp32(Assembler::LessThan, ToRegister(lir->index()), Imm32(min),
9147 0 : lir->snapshot());
9148 0 : }
9149 :
9150 : void
9151 0 : CodeGenerator::visitSpectreMaskIndex(LSpectreMaskIndex* lir)
9152 : {
9153 0 : MOZ_ASSERT(JitOptions.spectreIndexMasking);
9154 :
9155 53 : const LAllocation* length = lir->length();
9156 0 : Register index = ToRegister(lir->index());
9157 106 : Register output = ToRegister(lir->output());
9158 :
9159 53 : if (length->isRegister())
9160 0 : masm.spectreMaskIndex(index, ToRegister(length), output);
9161 : else
9162 0 : masm.spectreMaskIndex(index, ToAddress(length), output);
9163 53 : }
9164 :
9165 : class OutOfLineStoreElementHole : public OutOfLineCodeBase<CodeGenerator>
9166 : {
9167 : LInstruction* ins_;
9168 : Label rejoinStore_;
9169 : Label callStub_;
9170 : bool strict_;
9171 :
9172 : public:
9173 11 : explicit OutOfLineStoreElementHole(LInstruction* ins, bool strict)
9174 44 : : ins_(ins), strict_(strict)
9175 : {
9176 31 : MOZ_ASSERT(ins->isStoreElementHoleV() || ins->isStoreElementHoleT() ||
9177 : ins->isFallibleStoreElementV() || ins->isFallibleStoreElementT());
9178 0 : }
9179 :
9180 11 : void accept(CodeGenerator* codegen) override {
9181 0 : codegen->visitOutOfLineStoreElementHole(this);
9182 11 : }
9183 : LInstruction* ins() const {
9184 : return ins_;
9185 : }
9186 : Label* rejoinStore() {
9187 0 : return &rejoinStore_;
9188 : }
9189 : Label* callStub() {
9190 11 : return &callStub_;
9191 : }
9192 : bool strict() const {
9193 : return strict_;
9194 : }
9195 : };
9196 :
9197 : void
9198 0 : CodeGenerator::emitStoreHoleCheck(Register elements, const LAllocation* index,
9199 : int32_t offsetAdjustment, LSnapshot* snapshot)
9200 : {
9201 0 : Label bail;
9202 0 : if (index->isConstant()) {
9203 0 : Address dest(elements, ToInt32(index) * sizeof(js::Value) + offsetAdjustment);
9204 0 : masm.branchTestMagic(Assembler::Equal, dest, &bail);
9205 : } else {
9206 0 : BaseIndex dest(elements, ToRegister(index), TimesEight, offsetAdjustment);
9207 0 : masm.branchTestMagic(Assembler::Equal, dest, &bail);
9208 : }
9209 0 : bailoutFrom(&bail, snapshot);
9210 0 : }
9211 :
9212 : static ConstantOrRegister
9213 106 : ToConstantOrRegister(const LAllocation* value, MIRType valueType)
9214 : {
9215 0 : if (value->isConstant())
9216 56 : return ConstantOrRegister(value->toConstant()->toJSValue());
9217 234 : return TypedOrValueRegister(valueType, ToAnyRegister(value));
9218 : }
9219 :
9220 : void
9221 0 : CodeGenerator::emitStoreElementTyped(const LAllocation* value,
9222 : MIRType valueType, MIRType elementType,
9223 : Register elements, const LAllocation* index,
9224 : int32_t offsetAdjustment)
9225 : {
9226 0 : ConstantOrRegister v = ToConstantOrRegister(value, valueType);
9227 106 : if (index->isConstant()) {
9228 140 : Address dest(elements, ToInt32(index) * sizeof(js::Value) + offsetAdjustment);
9229 70 : masm.storeUnboxedValue(v, valueType, dest, elementType);
9230 : } else {
9231 0 : BaseIndex dest(elements, ToRegister(index), TimesEight, offsetAdjustment);
9232 0 : masm.storeUnboxedValue(v, valueType, dest, elementType);
9233 : }
9234 0 : }
9235 :
9236 : void
9237 0 : CodeGenerator::visitStoreElementT(LStoreElementT* store)
9238 : {
9239 0 : Register elements = ToRegister(store->elements());
9240 88 : const LAllocation* index = store->index();
9241 :
9242 0 : if (store->mir()->needsBarrier())
9243 36 : emitPreBarrier(elements, index, store->mir()->offsetAdjustment());
9244 :
9245 0 : if (store->mir()->needsHoleCheck())
9246 0 : emitStoreHoleCheck(elements, index, store->mir()->offsetAdjustment(), store->snapshot());
9247 :
9248 0 : emitStoreElementTyped(store->value(),
9249 88 : store->mir()->value()->type(), store->mir()->elementType(),
9250 0 : elements, index, store->mir()->offsetAdjustment());
9251 0 : }
9252 :
9253 : void
9254 0 : CodeGenerator::visitStoreElementV(LStoreElementV* lir)
9255 : {
9256 0 : const ValueOperand value = ToValue(lir, LStoreElementV::Value);
9257 60 : Register elements = ToRegister(lir->elements());
9258 30 : const LAllocation* index = lir->index();
9259 :
9260 60 : if (lir->mir()->needsBarrier())
9261 0 : emitPreBarrier(elements, index, lir->mir()->offsetAdjustment());
9262 :
9263 0 : if (lir->mir()->needsHoleCheck())
9264 0 : emitStoreHoleCheck(elements, index, lir->mir()->offsetAdjustment(), lir->snapshot());
9265 :
9266 0 : if (lir->index()->isConstant()) {
9267 : Address dest(elements,
9268 0 : ToInt32(lir->index()) * sizeof(js::Value) + lir->mir()->offsetAdjustment());
9269 0 : masm.storeValue(value, dest);
9270 : } else {
9271 : BaseIndex dest(elements, ToRegister(lir->index()), TimesEight,
9272 0 : lir->mir()->offsetAdjustment());
9273 0 : masm.storeValue(value, dest);
9274 : }
9275 30 : }
9276 :
9277 : template <typename T> void
9278 0 : CodeGenerator::emitStoreElementHoleT(T* lir)
9279 : {
9280 : static_assert(std::is_same<T, LStoreElementHoleT>::value || std::is_same<T, LFallibleStoreElementT>::value,
9281 : "emitStoreElementHoleT called with unexpected argument type");
9282 :
9283 : OutOfLineStoreElementHole* ool =
9284 36 : new(alloc()) OutOfLineStoreElementHole(lir, current->mir()->strict());
9285 18 : addOutOfLineCode(ool, lir->mir());
9286 :
9287 18 : Register elements = ToRegister(lir->elements());
9288 18 : Register index = ToRegister(lir->index());
9289 0 : Register spectreTemp = ToTempRegisterOrInvalid(lir->spectreTemp());
9290 :
9291 18 : Address initLength(elements, ObjectElements::offsetOfInitializedLength());
9292 0 : masm.spectreBoundsCheck32(index, initLength, spectreTemp, ool->entry());
9293 :
9294 0 : if (lir->mir()->needsBarrier())
9295 9 : emitPreBarrier(elements, lir->index(), 0);
9296 :
9297 : if (std::is_same<T, LFallibleStoreElementT>::value) {
9298 : // If the object might be non-extensible, check for frozen elements and
9299 : // holes.
9300 0 : Address flags(elements, ObjectElements::offsetOfFlags());
9301 0 : masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN),
9302 : ool->callStub());
9303 0 : if (lir->toFallibleStoreElementT()->mir()->needsHoleCheck()) {
9304 0 : masm.branchTestMagic(Assembler::Equal, BaseValueIndex(elements, index),
9305 : ool->callStub());
9306 : }
9307 : }
9308 :
9309 0 : masm.bind(ool->rejoinStore());
9310 45 : emitStoreElementTyped(lir->value(), lir->mir()->value()->type(), lir->mir()->elementType(),
9311 : elements, lir->index(), 0);
9312 :
9313 18 : masm.bind(ool->rejoin());
9314 0 : }
9315 :
9316 : void
9317 0 : CodeGenerator::visitStoreElementHoleT(LStoreElementHoleT* lir)
9318 : {
9319 0 : emitStoreElementHoleT(lir);
9320 0 : }
9321 :
9322 : template <typename T> void
9323 2 : CodeGenerator::emitStoreElementHoleV(T* lir)
9324 : {
9325 : static_assert(std::is_same<T, LStoreElementHoleV>::value || std::is_same<T, LFallibleStoreElementV>::value,
9326 : "emitStoreElementHoleV called with unexpected parameter type");
9327 :
9328 : OutOfLineStoreElementHole* ool =
9329 8 : new(alloc()) OutOfLineStoreElementHole(lir, current->mir()->strict());
9330 4 : addOutOfLineCode(ool, lir->mir());
9331 :
9332 4 : Register elements = ToRegister(lir->elements());
9333 4 : Register index = ToRegister(lir->index());
9334 0 : const ValueOperand value = ToValue(lir, T::Value);
9335 0 : Register spectreTemp = ToTempRegisterOrInvalid(lir->spectreTemp());
9336 :
9337 0 : Address initLength(elements, ObjectElements::offsetOfInitializedLength());
9338 0 : masm.spectreBoundsCheck32(index, initLength, spectreTemp, ool->entry());
9339 :
9340 : if (std::is_same<T, LFallibleStoreElementV>::value) {
9341 : // If the object might be non-extensible, check for frozen elements and
9342 : // holes.
9343 0 : Address flags(elements, ObjectElements::offsetOfFlags());
9344 0 : masm.branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::FROZEN),
9345 : ool->callStub());
9346 0 : if (lir->toFallibleStoreElementV()->mir()->needsHoleCheck()) {
9347 0 : masm.branchTestMagic(Assembler::Equal, BaseValueIndex(elements, index),
9348 : ool->callStub());
9349 : }
9350 : }
9351 :
9352 1 : if (lir->mir()->needsBarrier())
9353 2 : emitPreBarrier(elements, lir->index(), 0);
9354 :
9355 2 : masm.bind(ool->rejoinStore());
9356 2 : masm.storeValue(value, BaseIndex(elements, index, TimesEight));
9357 :
9358 0 : masm.bind(ool->rejoin());
9359 2 : }
9360 :
9361 : void
9362 0 : CodeGenerator::visitStoreElementHoleV(LStoreElementHoleV* lir)
9363 : {
9364 0 : emitStoreElementHoleV(lir);
9365 0 : }
9366 :
9367 : void
9368 0 : CodeGenerator::visitFallibleStoreElementT(LFallibleStoreElementT* lir)
9369 : {
9370 0 : emitStoreElementHoleT(lir);
9371 0 : }
9372 :
9373 : void
9374 0 : CodeGenerator::visitFallibleStoreElementV(LFallibleStoreElementV* lir)
9375 : {
9376 0 : emitStoreElementHoleV(lir);
9377 0 : }
9378 :
9379 : typedef bool (*SetDenseElementFn)(JSContext*, HandleNativeObject, int32_t, HandleValue,
9380 : bool strict);
9381 1 : static const VMFunction SetDenseElementInfo =
9382 1 : FunctionInfo<SetDenseElementFn>(jit::SetDenseElement, "SetDenseElement");
9383 :
9384 : void
9385 11 : CodeGenerator::visitOutOfLineStoreElementHole(OutOfLineStoreElementHole* ool)
9386 : {
9387 0 : Register object, elements;
9388 11 : LInstruction* ins = ool->ins();
9389 : const LAllocation* index;
9390 : MIRType valueType;
9391 11 : ConstantOrRegister value;
9392 : Register spectreTemp;
9393 :
9394 22 : if (ins->isStoreElementHoleV()) {
9395 2 : LStoreElementHoleV* store = ins->toStoreElementHoleV();
9396 0 : object = ToRegister(store->object());
9397 4 : elements = ToRegister(store->elements());
9398 2 : index = store->index();
9399 0 : valueType = store->mir()->value()->type();
9400 0 : value = TypedOrValueRegister(ToValue(store, LStoreElementHoleV::Value));
9401 0 : spectreTemp = ToTempRegisterOrInvalid(store->spectreTemp());
9402 0 : } else if (ins->isFallibleStoreElementV()) {
9403 0 : LFallibleStoreElementV* store = ins->toFallibleStoreElementV();
9404 0 : object = ToRegister(store->object());
9405 0 : elements = ToRegister(store->elements());
9406 0 : index = store->index();
9407 0 : valueType = store->mir()->value()->type();
9408 0 : value = TypedOrValueRegister(ToValue(store, LFallibleStoreElementV::Value));
9409 0 : spectreTemp = ToTempRegisterOrInvalid(store->spectreTemp());
9410 1 : } else if (ins->isStoreElementHoleT()) {
9411 1 : LStoreElementHoleT* store = ins->toStoreElementHoleT();
9412 1 : object = ToRegister(store->object());
9413 1 : elements = ToRegister(store->elements());
9414 1 : index = store->index();
9415 0 : valueType = store->mir()->value()->type();
9416 0 : if (store->value()->isConstant())
9417 0 : value = ConstantOrRegister(store->value()->toConstant()->toJSValue());
9418 : else
9419 0 : value = TypedOrValueRegister(valueType, ToAnyRegister(store->value()));
9420 0 : spectreTemp = ToTempRegisterOrInvalid(store->spectreTemp());
9421 : } else { // ins->isFallibleStoreElementT()
9422 0 : LFallibleStoreElementT* store = ins->toFallibleStoreElementT();
9423 0 : object = ToRegister(store->object());
9424 0 : elements = ToRegister(store->elements());
9425 0 : index = store->index();
9426 0 : valueType = store->mir()->value()->type();
9427 0 : if (store->value()->isConstant())
9428 0 : value = ConstantOrRegister(store->value()->toConstant()->toJSValue());
9429 : else
9430 0 : value = TypedOrValueRegister(valueType, ToAnyRegister(store->value()));
9431 0 : spectreTemp = ToTempRegisterOrInvalid(store->spectreTemp());
9432 : }
9433 :
9434 11 : Register indexReg = ToRegister(index);
9435 :
9436 : // If index == initializedLength, try to bump the initialized length inline.
9437 : // If index > initializedLength, call a stub. Note that this relies on the
9438 : // condition flags sticking from the incoming branch.
9439 : // Also note: this branch does not need Spectre mitigations, doing that for
9440 : // the capacity check below is sufficient.
9441 : #if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
9442 : // Had to reimplement for MIPS because there are no flags.
9443 : Address initLength(elements, ObjectElements::offsetOfInitializedLength());
9444 : masm.branch32(Assembler::NotEqual, initLength, indexReg, ool->callStub());
9445 : #else
9446 22 : masm.j(Assembler::NotEqual, ool->callStub());
9447 : #endif
9448 :
9449 : // Check array capacity.
9450 33 : masm.spectreBoundsCheck32(indexReg, Address(elements, ObjectElements::offsetOfCapacity()),
9451 0 : spectreTemp, ool->callStub());
9452 :
9453 : // Update initialized length. The capacity guard above ensures this won't overflow,
9454 : // due to MAX_DENSE_ELEMENTS_COUNT.
9455 0 : masm.add32(Imm32(1), indexReg);
9456 0 : masm.store32(indexReg, Address(elements, ObjectElements::offsetOfInitializedLength()));
9457 :
9458 : // Update length if length < initializedLength.
9459 22 : Label dontUpdate;
9460 0 : masm.branch32(Assembler::AboveOrEqual, Address(elements, ObjectElements::offsetOfLength()),
9461 0 : indexReg, &dontUpdate);
9462 22 : masm.store32(indexReg, Address(elements, ObjectElements::offsetOfLength()));
9463 11 : masm.bind(&dontUpdate);
9464 :
9465 0 : masm.sub32(Imm32(1), indexReg);
9466 :
9467 0 : if ((ins->isStoreElementHoleT() || ins->isFallibleStoreElementT()) &&
9468 : valueType != MIRType::Double)
9469 : {
9470 : // The inline path for StoreElementHoleT and FallibleStoreElementT does not always store
9471 : // the type tag, so we do the store on the OOL path. We use MIRType::None for the element
9472 : // type so that storeElementTyped will always store the type tag.
9473 18 : if (ins->isStoreElementHoleT()) {
9474 18 : emitStoreElementTyped(ins->toStoreElementHoleT()->value(), valueType, MIRType::None,
9475 9 : elements, index, 0);
9476 18 : masm.jump(ool->rejoin());
9477 0 : } else if (ins->isFallibleStoreElementT()) {
9478 0 : emitStoreElementTyped(ins->toFallibleStoreElementT()->value(), valueType,
9479 0 : MIRType::None, elements, index, 0);
9480 0 : masm.jump(ool->rejoin());
9481 : }
9482 : } else {
9483 : // Jump to the inline path where we will store the value.
9484 0 : masm.jump(ool->rejoinStore());
9485 : }
9486 :
9487 11 : masm.bind(ool->callStub());
9488 11 : saveLive(ins);
9489 :
9490 33 : pushArg(Imm32(ool->strict()));
9491 22 : pushArg(value);
9492 0 : if (index->isConstant())
9493 0 : pushArg(Imm32(ToInt32(index)));
9494 : else
9495 0 : pushArg(ToRegister(index));
9496 0 : pushArg(object);
9497 0 : callVM(SetDenseElementInfo, ins);
9498 :
9499 11 : restoreLive(ins);
9500 0 : masm.jump(ool->rejoin());
9501 0 : }
9502 :
9503 : template <typename T>
9504 : static void
9505 0 : StoreUnboxedPointer(MacroAssembler& masm, T address, MIRType type, const LAllocation* value,
9506 : bool preBarrier)
9507 : {
9508 6 : if (preBarrier)
9509 6 : masm.guardedCallPreBarrier(address, type);
9510 0 : if (value->isConstant()) {
9511 0 : Value v = value->toConstant()->toJSValue();
9512 0 : if (v.isGCThing()) {
9513 0 : masm.storePtr(ImmGCPtr(v.toGCThing()), address);
9514 : } else {
9515 0 : MOZ_ASSERT(v.isNull());
9516 0 : masm.storePtr(ImmWord(0), address);
9517 : }
9518 : } else {
9519 6 : masm.storePtr(ToRegister(value), address);
9520 : }
9521 0 : }
9522 :
9523 : void
9524 0 : CodeGenerator::visitStoreUnboxedPointer(LStoreUnboxedPointer* lir)
9525 : {
9526 : MIRType type;
9527 : int32_t offsetAdjustment;
9528 : bool preBarrier;
9529 0 : if (lir->mir()->isStoreUnboxedObjectOrNull()) {
9530 0 : type = MIRType::Object;
9531 0 : offsetAdjustment = lir->mir()->toStoreUnboxedObjectOrNull()->offsetAdjustment();
9532 0 : preBarrier = lir->mir()->toStoreUnboxedObjectOrNull()->preBarrier();
9533 12 : } else if (lir->mir()->isStoreUnboxedString()) {
9534 0 : type = MIRType::String;
9535 0 : offsetAdjustment = lir->mir()->toStoreUnboxedString()->offsetAdjustment();
9536 0 : preBarrier = lir->mir()->toStoreUnboxedString()->preBarrier();
9537 : } else {
9538 0 : MOZ_CRASH();
9539 : }
9540 :
9541 0 : Register elements = ToRegister(lir->elements());
9542 6 : const LAllocation* index = lir->index();
9543 1 : const LAllocation* value = lir->value();
9544 :
9545 6 : if (index->isConstant()) {
9546 0 : Address address(elements, ToInt32(index) * sizeof(uintptr_t) + offsetAdjustment);
9547 0 : StoreUnboxedPointer(masm, address, type, value, preBarrier);
9548 : } else {
9549 0 : BaseIndex address(elements, ToRegister(index), ScalePointer, offsetAdjustment);
9550 0 : StoreUnboxedPointer(masm, address, type, value, preBarrier);
9551 : }
9552 0 : }
9553 :
9554 : typedef NativeObject* (*ConvertUnboxedObjectToNativeFn)(JSContext*, JSObject*);
9555 1 : static const VMFunction ConvertUnboxedPlainObjectToNativeInfo =
9556 2 : FunctionInfo<ConvertUnboxedObjectToNativeFn>(UnboxedPlainObject::convertToNative,
9557 0 : "UnboxedPlainObject::convertToNative");
9558 :
9559 : void
9560 0 : CodeGenerator::visitConvertUnboxedObjectToNative(LConvertUnboxedObjectToNative* lir)
9561 : {
9562 0 : Register object = ToRegister(lir->getOperand(0));
9563 2 : Register temp = ToTempRegisterOrInvalid(lir->temp());
9564 :
9565 : // The call will return the same object so StoreRegisterTo(object) is safe.
9566 2 : OutOfLineCode* ool = oolCallVM(ConvertUnboxedPlainObjectToNativeInfo,
9567 0 : lir, ArgList(object), StoreRegisterTo(object));
9568 :
9569 6 : masm.branchTestObjGroup(Assembler::Equal, object, lir->mir()->group(), temp, object,
9570 2 : ool->entry());
9571 0 : masm.bind(ool->rejoin());
9572 0 : }
9573 :
9574 : typedef bool (*ArrayPopShiftFn)(JSContext*, HandleObject, MutableHandleValue);
9575 0 : static const VMFunction ArrayPopDenseInfo =
9576 0 : FunctionInfo<ArrayPopShiftFn>(jit::ArrayPopDense, "ArrayPopDense");
9577 0 : static const VMFunction ArrayShiftDenseInfo =
9578 3 : FunctionInfo<ArrayPopShiftFn>(jit::ArrayShiftDense, "ArrayShiftDense");
9579 :
9580 : void
9581 0 : CodeGenerator::emitArrayPopShift(LInstruction* lir, const MArrayPopShift* mir, Register obj,
9582 : Register elementsTemp, Register lengthTemp, TypedOrValueRegister out)
9583 : {
9584 : OutOfLineCode* ool;
9585 :
9586 0 : if (mir->mode() == MArrayPopShift::Pop) {
9587 0 : ool = oolCallVM(ArrayPopDenseInfo, lir, ArgList(obj), StoreValueTo(out));
9588 : } else {
9589 0 : MOZ_ASSERT(mir->mode() == MArrayPopShift::Shift);
9590 0 : ool = oolCallVM(ArrayShiftDenseInfo, lir, ArgList(obj), StoreValueTo(out));
9591 : }
9592 :
9593 : // VM call if a write barrier is necessary.
9594 0 : masm.branchTestNeedsIncrementalBarrier(Assembler::NonZero, ool->entry());
9595 :
9596 : // Load elements and initializedLength, and VM call if
9597 : // length != initializedLength.
9598 0 : masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp);
9599 0 : masm.load32(Address(elementsTemp, ObjectElements::offsetOfInitializedLength()), lengthTemp);
9600 :
9601 0 : Address lengthAddr(elementsTemp, ObjectElements::offsetOfLength());
9602 0 : masm.branch32(Assembler::NotEqual, lengthAddr, lengthTemp, ool->entry());
9603 :
9604 : // Test for length != 0. On zero length either take a VM call or generate
9605 : // an undefined value, depending on whether the call is known to produce
9606 : // undefined.
9607 0 : Label done;
9608 0 : if (mir->maybeUndefined()) {
9609 0 : Label notEmpty;
9610 0 : masm.branchTest32(Assembler::NonZero, lengthTemp, lengthTemp, ¬Empty);
9611 :
9612 : // According to the spec we need to set the length 0 (which is already 0).
9613 : // This is observable when the array length is made non-writable.
9614 : // Handle this case in the OOL.
9615 0 : Address elementFlags(elementsTemp, ObjectElements::offsetOfFlags());
9616 0 : Imm32 bit(ObjectElements::NONWRITABLE_ARRAY_LENGTH);
9617 0 : masm.branchTest32(Assembler::NonZero, elementFlags, bit, ool->entry());
9618 :
9619 0 : masm.moveValue(UndefinedValue(), out.valueReg());
9620 0 : masm.jump(&done);
9621 0 : masm.bind(¬Empty);
9622 : } else {
9623 0 : masm.branchTest32(Assembler::Zero, lengthTemp, lengthTemp, ool->entry());
9624 : }
9625 :
9626 0 : masm.sub32(Imm32(1), lengthTemp);
9627 :
9628 0 : if (mir->mode() == MArrayPopShift::Pop) {
9629 0 : BaseIndex addr(elementsTemp, lengthTemp, TimesEight);
9630 0 : masm.loadElementTypedOrValue(addr, out, mir->needsHoleCheck(), ool->entry());
9631 : } else {
9632 0 : MOZ_ASSERT(mir->mode() == MArrayPopShift::Shift);
9633 0 : Address addr(elementsTemp, 0);
9634 0 : masm.loadElementTypedOrValue(addr, out, mir->needsHoleCheck(), ool->entry());
9635 : }
9636 :
9637 : // Handle the failure cases when the array length is non-writable or the
9638 : // object is sealed in the OOL path. (Unlike in the adding-an-element cases,
9639 : // we can't rely on the capacity <= length invariant for such arrays to
9640 : // avoid an explicit check.)
9641 0 : Address elementFlags(elementsTemp, ObjectElements::offsetOfFlags());
9642 0 : Imm32 bits(ObjectElements::NONWRITABLE_ARRAY_LENGTH | ObjectElements::SEALED);
9643 0 : masm.branchTest32(Assembler::NonZero, elementFlags, bits, ool->entry());
9644 :
9645 0 : if (mir->mode() == MArrayPopShift::Shift) {
9646 : // Don't save the elementsTemp register.
9647 0 : LiveRegisterSet temps;
9648 0 : temps.add(elementsTemp);
9649 :
9650 0 : saveVolatile(temps);
9651 0 : masm.setupUnalignedABICall(elementsTemp);
9652 0 : masm.passABIArg(obj);
9653 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::ArrayShiftMoveElements));
9654 0 : restoreVolatile(temps);
9655 :
9656 : // Reload elementsTemp as ArrayShiftMoveElements may have moved it.
9657 0 : masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp);
9658 : }
9659 :
9660 : // Now adjust length and initializedLength.
9661 0 : masm.store32(lengthTemp, Address(elementsTemp, ObjectElements::offsetOfLength()));
9662 0 : masm.store32(lengthTemp, Address(elementsTemp, ObjectElements::offsetOfInitializedLength()));
9663 :
9664 0 : masm.bind(&done);
9665 0 : masm.bind(ool->rejoin());
9666 0 : }
9667 :
9668 : void
9669 0 : CodeGenerator::visitArrayPopShiftV(LArrayPopShiftV* lir)
9670 : {
9671 0 : Register obj = ToRegister(lir->object());
9672 0 : Register elements = ToRegister(lir->temp0());
9673 0 : Register length = ToRegister(lir->temp1());
9674 0 : TypedOrValueRegister out(ToOutValue(lir));
9675 0 : emitArrayPopShift(lir, lir->mir(), obj, elements, length, out);
9676 0 : }
9677 :
9678 : void
9679 0 : CodeGenerator::visitArrayPopShiftT(LArrayPopShiftT* lir)
9680 : {
9681 0 : Register obj = ToRegister(lir->object());
9682 0 : Register elements = ToRegister(lir->temp0());
9683 0 : Register length = ToRegister(lir->temp1());
9684 0 : TypedOrValueRegister out(lir->mir()->type(), ToAnyRegister(lir->output()));
9685 0 : emitArrayPopShift(lir, lir->mir(), obj, elements, length, out);
9686 0 : }
9687 :
9688 : typedef bool (*ArrayPushDenseFn)(JSContext*, HandleArrayObject, HandleValue, uint32_t*);
9689 0 : static const VMFunction ArrayPushDenseInfo =
9690 0 : FunctionInfo<ArrayPushDenseFn>(jit::ArrayPushDense, "ArrayPushDense");
9691 :
9692 : void
9693 9 : CodeGenerator::emitArrayPush(LInstruction* lir, Register obj,
9694 : const ConstantOrRegister& value, Register elementsTemp, Register length,
9695 : Register spectreTemp)
9696 : {
9697 18 : OutOfLineCode* ool = oolCallVM(ArrayPushDenseInfo, lir, ArgList(obj, value), StoreRegisterTo(length));
9698 :
9699 : // Load elements and length.
9700 18 : masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp);
9701 18 : masm.load32(Address(elementsTemp, ObjectElements::offsetOfLength()), length);
9702 :
9703 : // Guard length == initializedLength.
9704 18 : Address initLength(elementsTemp, ObjectElements::offsetOfInitializedLength());
9705 0 : masm.branch32(Assembler::NotEqual, initLength, length, ool->entry());
9706 :
9707 : // Guard length < capacity.
9708 18 : Address capacity(elementsTemp, ObjectElements::offsetOfCapacity());
9709 0 : masm.spectreBoundsCheck32(length, capacity, spectreTemp, ool->entry());
9710 :
9711 : // Do the store.
9712 9 : masm.storeConstantOrRegister(value, BaseIndex(elementsTemp, length, TimesEight));
9713 :
9714 0 : masm.add32(Imm32(1), length);
9715 :
9716 : // Update length and initialized length.
9717 0 : masm.store32(length, Address(elementsTemp, ObjectElements::offsetOfLength()));
9718 18 : masm.store32(length, Address(elementsTemp, ObjectElements::offsetOfInitializedLength()));
9719 :
9720 9 : masm.bind(ool->rejoin());
9721 9 : }
9722 :
9723 : void
9724 0 : CodeGenerator::visitArrayPushV(LArrayPushV* lir)
9725 : {
9726 0 : Register obj = ToRegister(lir->object());
9727 0 : Register elementsTemp = ToRegister(lir->temp());
9728 0 : Register length = ToRegister(lir->output());
9729 0 : ConstantOrRegister value = TypedOrValueRegister(ToValue(lir, LArrayPushV::Value));
9730 0 : Register spectreTemp = ToTempRegisterOrInvalid(lir->spectreTemp());
9731 0 : emitArrayPush(lir, obj, value, elementsTemp, length, spectreTemp);
9732 0 : }
9733 :
9734 : void
9735 0 : CodeGenerator::visitArrayPushT(LArrayPushT* lir)
9736 : {
9737 0 : Register obj = ToRegister(lir->object());
9738 18 : Register elementsTemp = ToRegister(lir->temp());
9739 18 : Register length = ToRegister(lir->output());
9740 0 : ConstantOrRegister value;
9741 18 : if (lir->value()->isConstant())
9742 0 : value = ConstantOrRegister(lir->value()->toConstant()->toJSValue());
9743 : else
9744 0 : value = TypedOrValueRegister(lir->mir()->value()->type(), ToAnyRegister(lir->value()));
9745 0 : Register spectreTemp = ToTempRegisterOrInvalid(lir->spectreTemp());
9746 0 : emitArrayPush(lir, obj, value, elementsTemp, length, spectreTemp);
9747 0 : }
9748 :
9749 : typedef JSObject* (*ArraySliceDenseFn)(JSContext*, HandleObject, int32_t, int32_t, HandleObject);
9750 0 : static const VMFunction ArraySliceDenseInfo =
9751 0 : FunctionInfo<ArraySliceDenseFn>(array_slice_dense, "array_slice_dense");
9752 :
9753 : void
9754 0 : CodeGenerator::visitArraySlice(LArraySlice* lir)
9755 : {
9756 0 : Register object = ToRegister(lir->object());
9757 0 : Register begin = ToRegister(lir->begin());
9758 0 : Register end = ToRegister(lir->end());
9759 0 : Register temp1 = ToRegister(lir->temp1());
9760 0 : Register temp2 = ToRegister(lir->temp2());
9761 :
9762 0 : Label call, fail;
9763 :
9764 : // Try to allocate an object.
9765 0 : TemplateObject templateObject(lir->mir()->templateObj());
9766 0 : masm.createGCObject(temp1, temp2, templateObject, lir->mir()->initialHeap(), &fail);
9767 :
9768 : // Fixup the group of the result in case it doesn't match the template object.
9769 0 : masm.copyObjGroupNoPreBarrier(object, temp1, temp2);
9770 :
9771 0 : masm.jump(&call);
9772 : {
9773 0 : masm.bind(&fail);
9774 0 : masm.movePtr(ImmPtr(nullptr), temp1);
9775 : }
9776 0 : masm.bind(&call);
9777 :
9778 0 : pushArg(temp1);
9779 0 : pushArg(end);
9780 0 : pushArg(begin);
9781 0 : pushArg(object);
9782 0 : callVM(ArraySliceDenseInfo, lir);
9783 0 : }
9784 :
9785 : typedef JSString* (*ArrayJoinFn)(JSContext*, HandleObject, HandleString);
9786 0 : static const VMFunction ArrayJoinInfo = FunctionInfo<ArrayJoinFn>(jit::ArrayJoin, "ArrayJoin");
9787 :
9788 : void
9789 1 : CodeGenerator::visitArrayJoin(LArrayJoin* lir)
9790 : {
9791 0 : Label skipCall;
9792 :
9793 2 : Register output = ToRegister(lir->output());
9794 0 : Register sep = ToRegister(lir->separator());
9795 2 : Register array = ToRegister(lir->array());
9796 0 : if (lir->mir()->optimizeForArray()) {
9797 2 : Register temp = ToRegister(lir->temp());
9798 :
9799 0 : masm.loadPtr(Address(array, NativeObject::offsetOfElements()), temp);
9800 0 : Address length(temp, ObjectElements::offsetOfLength());
9801 0 : Address initLength(temp, ObjectElements::offsetOfInitializedLength());
9802 :
9803 : // Check for length == 0
9804 0 : Label notEmpty;
9805 0 : masm.branch32(Assembler::NotEqual, length, Imm32(0), ¬Empty);
9806 0 : const JSAtomState& names = GetJitContext()->runtime->names();
9807 3 : masm.movePtr(ImmGCPtr(names.empty), output);
9808 2 : masm.jump(&skipCall);
9809 :
9810 0 : masm.bind(¬Empty);
9811 0 : Label notSingleString;
9812 : // Check for length == 1, initializedLength >= 1, arr[0].isString()
9813 0 : masm.branch32(Assembler::NotEqual, length, Imm32(1), ¬SingleString);
9814 1 : masm.branch32(Assembler::LessThan, initLength, Imm32(1), ¬SingleString);
9815 :
9816 0 : Address elem0(temp, 0);
9817 2 : masm.branchTestString(Assembler::NotEqual, elem0, ¬SingleString);
9818 :
9819 : // At this point, 'output' can be used as a scratch register, since we're
9820 : // guaranteed to succeed.
9821 0 : masm.unboxString(elem0, output);
9822 0 : masm.jump(&skipCall);
9823 1 : masm.bind(¬SingleString);
9824 : }
9825 :
9826 0 : pushArg(sep);
9827 0 : pushArg(array);
9828 0 : callVM(ArrayJoinInfo, lir);
9829 1 : masm.bind(&skipCall);
9830 1 : }
9831 :
9832 : void
9833 0 : CodeGenerator::visitGetIteratorCache(LGetIteratorCache* lir)
9834 : {
9835 0 : LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
9836 : TypedOrValueRegister val =
9837 0 : toConstantOrRegister(lir, LGetIteratorCache::Value, lir->mir()->value()->type()).reg();
9838 0 : Register output = ToRegister(lir->output());
9839 0 : Register temp1 = ToRegister(lir->temp1());
9840 0 : Register temp2 = ToRegister(lir->temp2());
9841 :
9842 0 : IonGetIteratorIC ic(liveRegs, val, output, temp1, temp2);
9843 0 : addIC(lir, allocateIC(ic));
9844 0 : }
9845 :
9846 : static void
9847 0 : LoadNativeIterator(MacroAssembler& masm, Register obj, Register dest, Label* failures)
9848 : {
9849 0 : MOZ_ASSERT(obj != dest);
9850 :
9851 : // Test class.
9852 : masm.branchTestObjClass(Assembler::NotEqual, obj, &PropertyIteratorObject::class_, dest,
9853 0 : obj, failures);
9854 :
9855 : // Load NativeIterator object.
9856 0 : masm.loadObjPrivate(obj, JSObject::ITER_CLASS_NFIXED_SLOTS, dest);
9857 0 : }
9858 :
9859 : typedef bool (*IteratorMoreFn)(JSContext*, HandleObject, MutableHandleValue);
9860 1 : static const VMFunction IteratorMoreInfo =
9861 0 : FunctionInfo<IteratorMoreFn>(IteratorMore, "IteratorMore");
9862 :
9863 : void
9864 0 : CodeGenerator::visitIteratorMore(LIteratorMore* lir)
9865 : {
9866 0 : const Register obj = ToRegister(lir->object());
9867 0 : const ValueOperand output = ToOutValue(lir);
9868 0 : const Register temp = ToRegister(lir->temp());
9869 :
9870 0 : OutOfLineCode* ool = oolCallVM(IteratorMoreInfo, lir, ArgList(obj), StoreValueTo(output));
9871 :
9872 0 : Register outputScratch = output.scratchReg();
9873 0 : LoadNativeIterator(masm, obj, outputScratch, ool->entry());
9874 :
9875 : // If propertyCursor_ < propertiesEnd_, load the next string and advance
9876 : // the cursor. Otherwise return MagicValue(JS_NO_ITER_VALUE).
9877 0 : Label iterDone;
9878 0 : Address cursorAddr(outputScratch, NativeIterator::offsetOfPropertyCursor());
9879 0 : Address cursorEndAddr(outputScratch, NativeIterator::offsetOfPropertiesEnd());
9880 0 : masm.loadPtr(cursorAddr, temp);
9881 0 : masm.branchPtr(Assembler::BelowOrEqual, cursorEndAddr, temp, &iterDone);
9882 :
9883 : // Get next string.
9884 0 : masm.loadPtr(Address(temp, 0), temp);
9885 :
9886 : // Increase the cursor.
9887 0 : masm.addPtr(Imm32(sizeof(GCPtrFlatString)), cursorAddr);
9888 :
9889 0 : masm.tagValue(JSVAL_TYPE_STRING, temp, output);
9890 0 : masm.jump(ool->rejoin());
9891 :
9892 0 : masm.bind(&iterDone);
9893 0 : masm.moveValue(MagicValue(JS_NO_ITER_VALUE), output);
9894 :
9895 0 : masm.bind(ool->rejoin());
9896 0 : }
9897 :
9898 : void
9899 0 : CodeGenerator::visitIsNoIterAndBranch(LIsNoIterAndBranch* lir)
9900 : {
9901 0 : ValueOperand input = ToValue(lir, LIsNoIterAndBranch::Input);
9902 0 : Label* ifTrue = getJumpLabelForBranch(lir->ifTrue());
9903 0 : Label* ifFalse = getJumpLabelForBranch(lir->ifFalse());
9904 :
9905 0 : masm.branchTestMagic(Assembler::Equal, input, ifTrue);
9906 :
9907 0 : if (!isNextBlock(lir->ifFalse()->lir()))
9908 0 : masm.jump(ifFalse);
9909 0 : }
9910 :
9911 : typedef void (*CloseIteratorFromIonFn)(JSContext*, JSObject*);
9912 0 : static const VMFunction CloseIteratorFromIonInfo =
9913 0 : FunctionInfo<CloseIteratorFromIonFn>(CloseIteratorFromIon, "CloseIteratorFromIon");
9914 :
9915 : void
9916 0 : CodeGenerator::visitIteratorEnd(LIteratorEnd* lir)
9917 : {
9918 0 : const Register obj = ToRegister(lir->object());
9919 0 : const Register temp1 = ToRegister(lir->temp1());
9920 0 : const Register temp2 = ToRegister(lir->temp2());
9921 0 : const Register temp3 = ToRegister(lir->temp3());
9922 :
9923 0 : OutOfLineCode* ool = oolCallVM(CloseIteratorFromIonInfo, lir, ArgList(obj), StoreNothing());
9924 :
9925 0 : LoadNativeIterator(masm, obj, temp1, ool->entry());
9926 :
9927 : // Clear active bit.
9928 0 : masm.and32(Imm32(~NativeIterator::Flags::Active),
9929 0 : Address(temp1, NativeIterator::offsetOfFlags()));
9930 :
9931 : // Reset property cursor.
9932 0 : masm.loadPtr(Address(temp1, NativeIterator::offsetOfGuardsEnd()), temp2);
9933 0 : masm.storePtr(temp2, Address(temp1, NativeIterator::offsetOfPropertyCursor()));
9934 :
9935 : // Unlink from the iterator list.
9936 0 : const Register next = temp2;
9937 0 : const Register prev = temp3;
9938 0 : masm.loadPtr(Address(temp1, NativeIterator::offsetOfNext()), next);
9939 0 : masm.loadPtr(Address(temp1, NativeIterator::offsetOfPrev()), prev);
9940 0 : masm.storePtr(prev, Address(next, NativeIterator::offsetOfPrev()));
9941 0 : masm.storePtr(next, Address(prev, NativeIterator::offsetOfNext()));
9942 : #ifdef DEBUG
9943 0 : masm.storePtr(ImmPtr(nullptr), Address(temp1, NativeIterator::offsetOfNext()));
9944 0 : masm.storePtr(ImmPtr(nullptr), Address(temp1, NativeIterator::offsetOfPrev()));
9945 : #endif
9946 :
9947 0 : masm.bind(ool->rejoin());
9948 0 : }
9949 :
9950 : void
9951 2 : CodeGenerator::visitArgumentsLength(LArgumentsLength* lir)
9952 : {
9953 : // read number of actual arguments from the JS frame.
9954 4 : Register argc = ToRegister(lir->output());
9955 6 : Address ptr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfNumActualArgs());
9956 :
9957 2 : masm.loadPtr(ptr, argc);
9958 2 : }
9959 :
9960 : void
9961 2 : CodeGenerator::visitGetFrameArgument(LGetFrameArgument* lir)
9962 : {
9963 0 : ValueOperand result = ToOutValue(lir);
9964 2 : const LAllocation* index = lir->index();
9965 4 : size_t argvOffset = frameSize() + JitFrameLayout::offsetOfActualArgs();
9966 :
9967 2 : if (index->isConstant()) {
9968 0 : int32_t i = index->toConstant()->toInt32();
9969 0 : Address argPtr(masm.getStackPointer(), sizeof(Value) * i + argvOffset);
9970 0 : masm.loadValue(argPtr, result);
9971 : } else {
9972 0 : Register i = ToRegister(index);
9973 1 : BaseValueIndex argPtr(masm.getStackPointer(), i, argvOffset);
9974 1 : masm.loadValue(argPtr, result);
9975 : }
9976 2 : }
9977 :
9978 : void
9979 0 : CodeGenerator::visitSetFrameArgumentT(LSetFrameArgumentT* lir)
9980 : {
9981 0 : size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() +
9982 0 : (sizeof(Value) * lir->mir()->argno());
9983 :
9984 0 : MIRType type = lir->mir()->value()->type();
9985 :
9986 0 : if (type == MIRType::Double) {
9987 : // Store doubles directly.
9988 0 : FloatRegister input = ToFloatRegister(lir->input());
9989 0 : masm.storeDouble(input, Address(masm.getStackPointer(), argOffset));
9990 :
9991 : } else {
9992 0 : Register input = ToRegister(lir->input());
9993 0 : masm.storeValue(ValueTypeFromMIRType(type), input, Address(masm.getStackPointer(), argOffset));
9994 : }
9995 0 : }
9996 :
9997 : void
9998 0 : CodeGenerator:: visitSetFrameArgumentC(LSetFrameArgumentC* lir)
9999 : {
10000 0 : size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() +
10001 0 : (sizeof(Value) * lir->mir()->argno());
10002 0 : masm.storeValue(lir->val(), Address(masm.getStackPointer(), argOffset));
10003 0 : }
10004 :
10005 : void
10006 0 : CodeGenerator:: visitSetFrameArgumentV(LSetFrameArgumentV* lir)
10007 : {
10008 0 : const ValueOperand val = ToValue(lir, LSetFrameArgumentV::Input);
10009 0 : size_t argOffset = frameSize() + JitFrameLayout::offsetOfActualArgs() +
10010 0 : (sizeof(Value) * lir->mir()->argno());
10011 0 : masm.storeValue(val, Address(masm.getStackPointer(), argOffset));
10012 0 : }
10013 :
10014 : typedef bool (*RunOnceScriptPrologueFn)(JSContext*, HandleScript);
10015 1 : static const VMFunction RunOnceScriptPrologueInfo =
10016 1 : FunctionInfo<RunOnceScriptPrologueFn>(js::RunOnceScriptPrologue, "RunOnceScriptPrologue");
10017 :
10018 : void
10019 0 : CodeGenerator::visitRunOncePrologue(LRunOncePrologue* lir)
10020 : {
10021 0 : pushArg(ImmGCPtr(lir->mir()->block()->info().script()));
10022 0 : callVM(RunOnceScriptPrologueInfo, lir);
10023 0 : }
10024 :
10025 : typedef JSObject* (*InitRestParameterFn)(JSContext*, uint32_t, Value*, HandleObject,
10026 : HandleObject);
10027 0 : static const VMFunction InitRestParameterInfo =
10028 0 : FunctionInfo<InitRestParameterFn>(InitRestParameter, "InitRestParameter");
10029 :
10030 : void
10031 0 : CodeGenerator::emitRest(LInstruction* lir, Register array, Register numActuals,
10032 : Register temp0, Register temp1, unsigned numFormals,
10033 : JSObject* templateObject, bool saveAndRestore, Register resultreg)
10034 : {
10035 : // Compute actuals() + numFormals.
10036 0 : size_t actualsOffset = frameSize() + JitFrameLayout::offsetOfActualArgs();
10037 0 : masm.moveStackPtrTo(temp1);
10038 0 : masm.addPtr(Imm32(sizeof(Value) * numFormals + actualsOffset), temp1);
10039 :
10040 : // Compute numActuals - numFormals.
10041 0 : Label emptyLength, joinLength;
10042 0 : masm.movePtr(numActuals, temp0);
10043 0 : masm.branch32(Assembler::LessThanOrEqual, temp0, Imm32(numFormals), &emptyLength);
10044 0 : masm.sub32(Imm32(numFormals), temp0);
10045 0 : masm.jump(&joinLength);
10046 : {
10047 0 : masm.bind(&emptyLength);
10048 0 : masm.move32(Imm32(0), temp0);
10049 : }
10050 0 : masm.bind(&joinLength);
10051 :
10052 0 : if (saveAndRestore)
10053 0 : saveLive(lir);
10054 :
10055 0 : pushArg(array);
10056 0 : pushArg(ImmGCPtr(templateObject));
10057 0 : pushArg(temp1);
10058 0 : pushArg(temp0);
10059 :
10060 0 : callVM(InitRestParameterInfo, lir);
10061 :
10062 0 : if (saveAndRestore) {
10063 0 : storePointerResultTo(resultreg);
10064 0 : restoreLive(lir);
10065 : }
10066 0 : }
10067 :
10068 : void
10069 0 : CodeGenerator::visitRest(LRest* lir)
10070 : {
10071 0 : Register numActuals = ToRegister(lir->numActuals());
10072 0 : Register temp0 = ToRegister(lir->getTemp(0));
10073 0 : Register temp1 = ToRegister(lir->getTemp(1));
10074 0 : Register temp2 = ToRegister(lir->getTemp(2));
10075 0 : unsigned numFormals = lir->mir()->numFormals();
10076 0 : ArrayObject* templateObject = lir->mir()->templateObject();
10077 :
10078 0 : Label joinAlloc, failAlloc;
10079 0 : TemplateObject templateObj(templateObject);
10080 0 : masm.createGCObject(temp2, temp0, templateObj, gc::DefaultHeap, &failAlloc);
10081 0 : masm.jump(&joinAlloc);
10082 : {
10083 0 : masm.bind(&failAlloc);
10084 0 : masm.movePtr(ImmPtr(nullptr), temp2);
10085 : }
10086 0 : masm.bind(&joinAlloc);
10087 :
10088 0 : emitRest(lir, temp2, numActuals, temp0, temp1, numFormals, templateObject, false, ToRegister(lir->output()));
10089 0 : }
10090 :
10091 : bool
10092 0 : CodeGenerator::generateWasm(wasm::SigIdDesc sigId, wasm::BytecodeOffset trapOffset,
10093 : wasm::FuncOffsets* offsets)
10094 : {
10095 0 : JitSpew(JitSpew_Codegen, "# Emitting wasm code");
10096 :
10097 0 : wasm::GenerateFunctionPrologue(masm, sigId, mozilla::Nothing(), offsets);
10098 :
10099 0 : if (omitOverRecursedCheck())
10100 0 : masm.reserveStack(frameSize());
10101 : else
10102 0 : masm.wasmReserveStackChecked(frameSize(), trapOffset);
10103 :
10104 0 : MOZ_ASSERT(masm.framePushed() == frameSize());
10105 :
10106 0 : if (!generateBody())
10107 : return false;
10108 :
10109 0 : masm.bind(&returnLabel_);
10110 0 : wasm::GenerateFunctionEpilogue(masm, frameSize(), offsets);
10111 :
10112 : #if defined(JS_ION_PERF)
10113 : // Note the end of the inline code and start of the OOL code.
10114 : gen->perfSpewer().noteEndInlineCode(masm);
10115 : #endif
10116 :
10117 0 : if (!generateOutOfLineCode())
10118 : return false;
10119 :
10120 0 : masm.flush();
10121 0 : if (masm.oom())
10122 : return false;
10123 :
10124 0 : offsets->end = masm.currentOffset();
10125 :
10126 0 : MOZ_ASSERT(!masm.failureLabel()->used());
10127 0 : MOZ_ASSERT(snapshots_.listSize() == 0);
10128 0 : MOZ_ASSERT(snapshots_.RVATableSize() == 0);
10129 0 : MOZ_ASSERT(recovers_.size() == 0);
10130 0 : MOZ_ASSERT(bailouts_.empty());
10131 0 : MOZ_ASSERT(graph.numConstants() == 0);
10132 0 : MOZ_ASSERT(safepointIndices_.empty());
10133 0 : MOZ_ASSERT(osiIndices_.empty());
10134 0 : MOZ_ASSERT(icList_.empty());
10135 0 : MOZ_ASSERT(safepoints_.size() == 0);
10136 0 : MOZ_ASSERT(!scriptCounts_);
10137 : return true;
10138 : }
10139 :
10140 : bool
10141 0 : CodeGenerator::generate()
10142 : {
10143 96 : JitSpew(JitSpew_Codegen, "# Emitting code for script %s:%u",
10144 48 : gen->info().script()->filename(),
10145 96 : gen->info().script()->lineno());
10146 :
10147 : // Initialize native code table with an entry to the start of
10148 : // top-level script.
10149 0 : InlineScriptTree* tree = gen->info().inlineScriptTree();
10150 0 : jsbytecode* startPC = tree->script()->code();
10151 96 : BytecodeSite* startSite = new(gen->alloc()) BytecodeSite(tree, startPC);
10152 48 : if (!addNativeToBytecodeEntry(startSite))
10153 : return false;
10154 :
10155 0 : if (!snapshots_.init())
10156 : return false;
10157 :
10158 48 : if (!safepoints_.init(gen->alloc()))
10159 : return false;
10160 :
10161 48 : if (!generatePrologue())
10162 : return false;
10163 :
10164 : // Before generating any code, we generate type checks for all parameters.
10165 : // This comes before deoptTable_, because we can't use deopt tables without
10166 : // creating the actual frame.
10167 48 : generateArgumentsChecks();
10168 :
10169 96 : if (frameClass_ != FrameSizeClass::None())
10170 0 : deoptTable_.emplace(gen->jitRuntime()->getBailoutTable(frameClass_));
10171 :
10172 : // Skip over the alternative entry to IonScript code.
10173 96 : Label skipPrologue;
10174 0 : masm.jump(&skipPrologue);
10175 :
10176 : // An alternative entry to the IonScript code, which doesn't test the
10177 : // arguments.
10178 0 : masm.flushBuffer();
10179 0 : setSkipArgCheckEntryOffset(masm.size());
10180 96 : masm.setFramePushed(0);
10181 48 : if (!generatePrologue())
10182 : return false;
10183 :
10184 0 : masm.bind(&skipPrologue);
10185 :
10186 : #ifdef DEBUG
10187 : // Assert that the argument types are correct.
10188 48 : generateArgumentsChecks(/* assert = */ true);
10189 : #endif
10190 :
10191 : // Reset native => bytecode map table with top-level script and startPc.
10192 48 : if (!addNativeToBytecodeEntry(startSite))
10193 : return false;
10194 :
10195 48 : if (!generateBody())
10196 : return false;
10197 :
10198 : // Reset native => bytecode map table with top-level script and startPc.
10199 48 : if (!addNativeToBytecodeEntry(startSite))
10200 : return false;
10201 :
10202 48 : if (!generateEpilogue())
10203 : return false;
10204 :
10205 : // Reset native => bytecode map table with top-level script and startPc.
10206 48 : if (!addNativeToBytecodeEntry(startSite))
10207 : return false;
10208 :
10209 48 : generateInvalidateEpilogue();
10210 : #if defined(JS_ION_PERF)
10211 : // Note the end of the inline code and start of the OOL code.
10212 : perfSpewer_.noteEndInlineCode(masm);
10213 : #endif
10214 :
10215 : // native => bytecode entries for OOL code will be added
10216 : // by CodeGeneratorShared::generateOutOfLineCode
10217 48 : if (!generateOutOfLineCode())
10218 : return false;
10219 :
10220 : // Add terminal entry.
10221 48 : if (!addNativeToBytecodeEntry(startSite))
10222 : return false;
10223 :
10224 : // Dump Native to bytecode entries to spew.
10225 48 : dumpNativeToBytecodeEntries();
10226 :
10227 96 : return !masm.oom();
10228 : }
10229 :
10230 : bool
10231 45 : CodeGenerator::linkSharedStubs(JSContext* cx)
10232 : {
10233 57 : for (uint32_t i = 0; i < sharedStubs_.length(); i++) {
10234 12 : ICStub *stub = nullptr;
10235 :
10236 0 : switch (sharedStubs_[i].kind) {
10237 : case ICStub::Kind::BinaryArith_Fallback: {
10238 0 : ICBinaryArith_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonSharedIC);
10239 0 : stub = stubCompiler.getStub(&stubSpace_);
10240 : break;
10241 : }
10242 : case ICStub::Kind::Compare_Fallback: {
10243 0 : ICCompare_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonSharedIC);
10244 0 : stub = stubCompiler.getStub(&stubSpace_);
10245 : break;
10246 : }
10247 : case ICStub::Kind::GetProp_Fallback: {
10248 0 : ICGetProp_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonSharedIC);
10249 0 : stub = stubCompiler.getStub(&stubSpace_);
10250 : break;
10251 : }
10252 : case ICStub::Kind::NewArray_Fallback: {
10253 1 : JSScript* script = sharedStubs_[i].entry.script();
10254 1 : jsbytecode* pc = sharedStubs_[i].entry.pc(script);
10255 8 : ObjectGroup* group = ObjectGroup::allocationSiteGroup(cx, script, pc, JSProto_Array);
10256 8 : if (!group)
10257 0 : return false;
10258 :
10259 0 : ICNewArray_Fallback::Compiler stubCompiler(cx, group, ICStubCompiler::Engine::IonSharedIC);
10260 0 : stub = stubCompiler.getStub(&stubSpace_);
10261 : break;
10262 : }
10263 : case ICStub::Kind::NewObject_Fallback: {
10264 0 : ICNewObject_Fallback::Compiler stubCompiler(cx, ICStubCompiler::Engine::IonSharedIC);
10265 0 : stub = stubCompiler.getStub(&stubSpace_);
10266 : break;
10267 : }
10268 : default:
10269 0 : MOZ_CRASH("Unsupported shared stub.");
10270 : }
10271 :
10272 12 : if (!stub)
10273 : return false;
10274 :
10275 24 : sharedStubs_[i].entry.setFirstStub(stub);
10276 : }
10277 : return true;
10278 : }
10279 :
10280 : bool
10281 45 : CodeGenerator::link(JSContext* cx, CompilerConstraintList* constraints)
10282 : {
10283 : // We cancel off-thread Ion compilations in a few places during GC, but if
10284 : // this compilation was performed off-thread it will already have been
10285 : // removed from the relevant lists by this point. Don't allow GC here.
10286 0 : JS::AutoAssertNoGC nogc(cx);
10287 :
10288 90 : RootedScript script(cx, gen->info().script());
10289 45 : OptimizationLevel optimizationLevel = gen->optimizationInfo().level();
10290 :
10291 : // Perform any read barriers which were skipped while compiling the
10292 : // script, which may have happened off-thread.
10293 0 : const JitRealm* jr = gen->realm->jitRealm();
10294 0 : jr->performStubReadBarriers(realmStubsToReadBarrier_);
10295 45 : jr->performSIMDTemplateReadBarriers(simdTemplatesToReadBarrier_);
10296 :
10297 : // We finished the new IonScript. Invalidate the current active IonScript,
10298 : // so we can replace it with this new (probably higher optimized) version.
10299 0 : if (script->hasIonScript()) {
10300 0 : MOZ_ASSERT(script->ionScript()->isRecompiling());
10301 : // Do a normal invalidate, except don't cancel offThread compilations,
10302 : // since that will cancel this compilation too.
10303 0 : Invalidate(cx, script, /* resetUses */ false, /* cancelOffThread*/ false);
10304 : }
10305 :
10306 45 : if (scriptCounts_ && !script->hasScriptCounts() && !script->initScriptCounts(cx))
10307 : return false;
10308 :
10309 45 : if (!linkSharedStubs(cx))
10310 : return false;
10311 :
10312 : // Check to make sure we didn't have a mid-build invalidation. If so, we
10313 : // will trickle to jit::Compile() and return Method_Skipped.
10314 0 : uint32_t warmUpCount = script->getWarmUpCount();
10315 :
10316 180 : IonCompilationId compilationId = cx->runtime()->jitRuntime()->nextCompilationId();
10317 90 : cx->zone()->types.currentCompilationIdRef().emplace(compilationId);
10318 45 : auto resetCurrentId = mozilla::MakeScopeExit([cx] {
10319 0 : cx->zone()->types.currentCompilationIdRef().reset();
10320 180 : });
10321 :
10322 : // Record constraints. If an error occured, returns false and potentially
10323 : // prevent future compilations. Otherwise, if an invalidation occured, then
10324 : // skip the current compilation.
10325 0 : bool isValid = false;
10326 45 : if (!FinishCompilation(cx, script, constraints, compilationId, &isValid))
10327 : return false;
10328 45 : if (!isValid)
10329 : return true;
10330 :
10331 : // IonMonkey could have inferred better type information during
10332 : // compilation. Since adding the new information to the actual type
10333 : // information can reset the usecount, increase it back to what it was
10334 : // before.
10335 78 : if (warmUpCount > script->getWarmUpCount())
10336 0 : script->incWarmUpCounter(warmUpCount - script->getWarmUpCount());
10337 :
10338 39 : uint32_t argumentSlots = (gen->info().nargs() + 1) * sizeof(Value);
10339 78 : uint32_t scriptFrameSize = frameClass_ == FrameSizeClass::None()
10340 0 : ? frameDepth_
10341 0 : : FrameSizeClass::FromDepth(frameDepth_).frameSize();
10342 :
10343 : // We encode safepoints after the OSI-point offsets have been determined.
10344 0 : if (!encodeSafepoints())
10345 : return false;
10346 :
10347 : IonScript* ionScript =
10348 273 : IonScript::New(cx, compilationId,
10349 0 : graph.totalSlotCount(), argumentSlots, scriptFrameSize,
10350 : snapshots_.listSize(), snapshots_.RVATableSize(),
10351 39 : recovers_.size(), bailouts_.length(), graph.numConstants(),
10352 : safepointIndices_.length(), osiIndices_.length(),
10353 : icList_.length(), runtimeData_.length(),
10354 : safepoints_.size(), sharedStubs_.length(),
10355 39 : optimizationLevel);
10356 0 : if (!ionScript)
10357 : return false;
10358 0 : auto guardIonScript = mozilla::MakeScopeExit([&ionScript] {
10359 : // Use js_free instead of IonScript::Destroy: the cache list is still
10360 : // uninitialized.
10361 0 : js_free(ionScript);
10362 117 : });
10363 :
10364 117 : Linker linker(masm);
10365 78 : AutoFlushICache afc("IonLink");
10366 0 : JitCode* code = linker.newCode(cx, CodeKind::Ion);
10367 0 : if (!code)
10368 : return false;
10369 :
10370 : // Encode native to bytecode map if profiling is enabled.
10371 0 : if (isProfilerInstrumentationEnabled()) {
10372 : // Generate native-to-bytecode main table.
10373 0 : if (!generateCompactNativeToBytecodeMap(cx, code))
10374 0 : return false;
10375 :
10376 0 : uint8_t* ionTableAddr = ((uint8_t*) nativeToBytecodeMap_) + nativeToBytecodeTableOffset_;
10377 0 : JitcodeIonTable* ionTable = (JitcodeIonTable*) ionTableAddr;
10378 :
10379 : // Construct the IonEntry that will go into the global table.
10380 : JitcodeGlobalEntry::IonEntry entry;
10381 0 : if (!ionTable->makeIonEntry(cx, code, nativeToBytecodeScriptListLength_,
10382 : nativeToBytecodeScriptList_, entry))
10383 : {
10384 0 : js_free(nativeToBytecodeScriptList_);
10385 0 : js_free(nativeToBytecodeMap_);
10386 0 : return false;
10387 : }
10388 :
10389 : // nativeToBytecodeScriptList_ is no longer needed.
10390 0 : js_free(nativeToBytecodeScriptList_);
10391 :
10392 : // Generate the tracked optimizations map.
10393 0 : if (isOptimizationTrackingEnabled()) {
10394 : // Treat OOMs and failures as if optimization tracking were turned off.
10395 0 : IonTrackedTypeVector* allTypes = cx->new_<IonTrackedTypeVector>();
10396 0 : if (allTypes && generateCompactTrackedOptimizationsMap(cx, code, allTypes)) {
10397 0 : const uint8_t* optsRegionTableAddr = trackedOptimizationsMap_ +
10398 0 : trackedOptimizationsRegionTableOffset_;
10399 : const IonTrackedOptimizationsRegionTable* optsRegionTable =
10400 0 : (const IonTrackedOptimizationsRegionTable*) optsRegionTableAddr;
10401 : const uint8_t* optsTypesTableAddr = trackedOptimizationsMap_ +
10402 0 : trackedOptimizationsTypesTableOffset_;
10403 : const IonTrackedOptimizationsTypesTable* optsTypesTable =
10404 0 : (const IonTrackedOptimizationsTypesTable*) optsTypesTableAddr;
10405 : const uint8_t* optsAttemptsTableAddr = trackedOptimizationsMap_ +
10406 0 : trackedOptimizationsAttemptsTableOffset_;
10407 : const IonTrackedOptimizationsAttemptsTable* optsAttemptsTable =
10408 0 : (const IonTrackedOptimizationsAttemptsTable*) optsAttemptsTableAddr;
10409 : entry.initTrackedOptimizations(optsRegionTable, optsTypesTable, optsAttemptsTable,
10410 : allTypes);
10411 : } else {
10412 0 : cx->recoverFromOutOfMemory();
10413 : }
10414 : }
10415 :
10416 : // Add entry to the global table.
10417 0 : JitcodeGlobalTable* globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
10418 0 : if (!globalTable->addEntry(entry)) {
10419 : // Memory may have been allocated for the entry.
10420 0 : entry.destroy();
10421 0 : return false;
10422 : }
10423 :
10424 : // Mark the jitcode as having a bytecode map.
10425 0 : code->setHasBytecodeMap();
10426 : } else {
10427 : // Add a dumy jitcodeGlobalTable entry.
10428 : JitcodeGlobalEntry::DummyEntry entry;
10429 117 : entry.init(code, code->raw(), code->rawEnd());
10430 :
10431 : // Add entry to the global table.
10432 117 : JitcodeGlobalTable* globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
10433 39 : if (!globalTable->addEntry(entry)) {
10434 : // Memory may have been allocated for the entry.
10435 0 : entry.destroy();
10436 0 : return false;
10437 : }
10438 :
10439 : // Mark the jitcode as having a bytecode map.
10440 0 : code->setHasBytecodeMap();
10441 : }
10442 :
10443 39 : ionScript->setMethod(code);
10444 39 : ionScript->setSkipArgCheckEntryOffset(getSkipArgCheckEntryOffset());
10445 :
10446 : // If the Gecko Profiler is enabled, mark IonScript as having been
10447 : // instrumented accordingly.
10448 0 : if (isProfilerInstrumentationEnabled())
10449 0 : ionScript->setHasProfilingInstrumentation();
10450 :
10451 117 : script->setIonScript(cx->runtime(), ionScript);
10452 :
10453 : // Adopt fallback shared stubs from the compiler into the ion script.
10454 0 : ionScript->adoptFallbackStubs(&stubSpace_);
10455 :
10456 0 : Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, invalidateEpilogueData_),
10457 : ImmPtr(ionScript),
10458 39 : ImmPtr((void*)-1));
10459 :
10460 1989 : for (size_t i = 0; i < ionScriptLabels_.length(); i++) {
10461 0 : Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, ionScriptLabels_[i]),
10462 : ImmPtr(ionScript),
10463 0 : ImmPtr((void*)-1));
10464 : }
10465 :
10466 : #ifdef JS_TRACE_LOGGING
10467 : bool TLFailed = false;
10468 :
10469 39 : for (uint32_t i = 0; i < patchableTLEvents_.length(); i++) {
10470 0 : TraceLoggerEvent event(patchableTLEvents_[i].event);
10471 0 : if (!event.hasTextId() || !ionScript->addTraceLoggerEvent(event)) {
10472 0 : TLFailed = true;
10473 0 : break;
10474 : }
10475 0 : Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, patchableTLEvents_[i].offset),
10476 0 : ImmPtr((void*) uintptr_t(event.textId())),
10477 0 : ImmPtr((void*)0));
10478 : }
10479 :
10480 1 : if (!TLFailed && patchableTLScripts_.length() > 0) {
10481 0 : MOZ_ASSERT(TraceLogTextIdEnabled(TraceLogger_Scripts));
10482 0 : TraceLoggerEvent event(TraceLogger_Scripts, script);
10483 0 : if (!event.hasTextId() || !ionScript->addTraceLoggerEvent(event))
10484 0 : TLFailed = true;
10485 0 : if (!TLFailed) {
10486 0 : uint32_t textId = event.textId();
10487 0 : for (uint32_t i = 0; i < patchableTLScripts_.length(); i++) {
10488 0 : Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, patchableTLScripts_[i]),
10489 0 : ImmPtr((void*) uintptr_t(textId)),
10490 0 : ImmPtr((void*)0));
10491 : }
10492 : }
10493 : }
10494 : #endif
10495 :
10496 : // Patch shared stub IC loads using IC entries
10497 55 : for (size_t i = 0; i < sharedStubs_.length(); i++) {
10498 8 : CodeOffset label = sharedStubs_[i].label;
10499 :
10500 16 : IonICEntry& entry = ionScript->sharedStubList()[i];
10501 8 : entry = sharedStubs_[i].entry;
10502 0 : Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, label),
10503 : ImmPtr(&entry),
10504 8 : ImmPtr((void*)-1));
10505 :
10506 0 : MOZ_ASSERT(entry.hasStub());
10507 0 : MOZ_ASSERT(entry.firstStub()->isFallback());
10508 :
10509 0 : entry.firstStub()->toFallbackStub()->fixupICEntry(&entry);
10510 : }
10511 :
10512 : // for generating inline caches during the execution.
10513 39 : if (runtimeData_.length())
10514 0 : ionScript->copyRuntimeData(&runtimeData_[0]);
10515 39 : if (icList_.length())
10516 26 : ionScript->copyICEntries(&icList_[0]);
10517 :
10518 0 : for (size_t i = 0; i < icInfo_.length(); i++) {
10519 0 : IonIC& ic = ionScript->getICFromIndex(i);
10520 0 : Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, icInfo_[i].icOffsetForJump),
10521 0 : ImmPtr(ic.codeRawPtr()),
10522 312 : ImmPtr((void*)-1));
10523 0 : Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, icInfo_[i].icOffsetForPush),
10524 : ImmPtr(&ic),
10525 0 : ImmPtr((void*)-1));
10526 : }
10527 :
10528 0 : JitSpew(JitSpew_Codegen, "Created IonScript %p (raw %p)",
10529 78 : (void*) ionScript, (void*) code->raw());
10530 :
10531 39 : ionScript->setInvalidationEpilogueDataOffset(invalidateEpilogueData_.offset());
10532 78 : ionScript->setOsrPc(gen->info().osrPc());
10533 0 : ionScript->setOsrEntryOffset(getOsrEntryOffset());
10534 0 : ionScript->setInvalidationEpilogueOffset(invalidate_.offset());
10535 :
10536 : #if defined(JS_ION_PERF)
10537 : if (PerfEnabled())
10538 : perfSpewer_.writeProfile(script, code, masm);
10539 : #endif
10540 :
10541 : #ifdef MOZ_VTUNE
10542 : vtune::MarkScript(code, script, "ion");
10543 : #endif
10544 :
10545 : // for marking during GC.
10546 39 : if (safepointIndices_.length())
10547 37 : ionScript->copySafepointIndices(&safepointIndices_[0]);
10548 78 : if (safepoints_.size())
10549 37 : ionScript->copySafepoints(&safepoints_);
10550 :
10551 : // for reconvering from an Ion Frame.
10552 0 : if (bailouts_.length())
10553 0 : ionScript->copyBailoutTable(&bailouts_[0]);
10554 0 : if (osiIndices_.length())
10555 39 : ionScript->copyOsiIndices(&osiIndices_[0]);
10556 78 : if (snapshots_.listSize())
10557 0 : ionScript->copySnapshots(&snapshots_);
10558 1 : MOZ_ASSERT_IF(snapshots_.listSize(), recovers_.size());
10559 0 : if (recovers_.size())
10560 0 : ionScript->copyRecovers(&recovers_);
10561 0 : if (graph.numConstants()) {
10562 0 : const Value* vp = graph.constantPool();
10563 0 : ionScript->copyConstants(vp);
10564 0 : for (size_t i = 0; i < graph.numConstants(); i++) {
10565 0 : const Value& v = vp[i];
10566 0 : if ((v.isObject() || v.isString()) && IsInsideNursery(v.toGCThing())) {
10567 0 : cx->runtime()->gc.storeBuffer().putWholeCell(script);
10568 : break;
10569 : }
10570 : }
10571 : }
10572 :
10573 : // Attach any generated script counts to the script.
10574 78 : if (IonScriptCounts* counts = extractScriptCounts())
10575 0 : script->addIonCounts(counts);
10576 :
10577 39 : guardIonScript.release();
10578 39 : return true;
10579 : }
10580 :
10581 : // An out-of-line path to convert a boxed int32 to either a float or double.
10582 : class OutOfLineUnboxFloatingPoint : public OutOfLineCodeBase<CodeGenerator>
10583 : {
10584 : LUnboxFloatingPoint* unboxFloatingPoint_;
10585 :
10586 : public:
10587 : explicit OutOfLineUnboxFloatingPoint(LUnboxFloatingPoint* unboxFloatingPoint)
10588 0 : : unboxFloatingPoint_(unboxFloatingPoint)
10589 : { }
10590 :
10591 0 : void accept(CodeGenerator* codegen) override {
10592 0 : codegen->visitOutOfLineUnboxFloatingPoint(this);
10593 0 : }
10594 :
10595 : LUnboxFloatingPoint* unboxFloatingPoint() const {
10596 : return unboxFloatingPoint_;
10597 : }
10598 : };
10599 :
10600 : void
10601 0 : CodeGenerator::visitUnboxFloatingPoint(LUnboxFloatingPoint* lir)
10602 : {
10603 0 : const ValueOperand box = ToValue(lir, LUnboxFloatingPoint::Input);
10604 0 : const LDefinition* result = lir->output();
10605 :
10606 : // Out-of-line path to convert int32 to double or bailout
10607 : // if this instruction is fallible.
10608 0 : OutOfLineUnboxFloatingPoint* ool = new(alloc()) OutOfLineUnboxFloatingPoint(lir);
10609 0 : addOutOfLineCode(ool, lir->mir());
10610 :
10611 0 : FloatRegister resultReg = ToFloatRegister(result);
10612 0 : masm.branchTestDouble(Assembler::NotEqual, box, ool->entry());
10613 0 : masm.unboxDouble(box, resultReg);
10614 0 : if (lir->type() == MIRType::Float32)
10615 0 : masm.convertDoubleToFloat32(resultReg, resultReg);
10616 0 : masm.bind(ool->rejoin());
10617 0 : }
10618 :
10619 : void
10620 0 : CodeGenerator::visitOutOfLineUnboxFloatingPoint(OutOfLineUnboxFloatingPoint* ool)
10621 : {
10622 0 : LUnboxFloatingPoint* ins = ool->unboxFloatingPoint();
10623 0 : const ValueOperand value = ToValue(ins, LUnboxFloatingPoint::Input);
10624 :
10625 0 : if (ins->mir()->fallible()) {
10626 0 : Label bail;
10627 0 : masm.branchTestInt32(Assembler::NotEqual, value, &bail);
10628 0 : bailoutFrom(&bail, ins->snapshot());
10629 : }
10630 0 : masm.int32ValueToFloatingPoint(value, ToFloatRegister(ins->output()), ins->type());
10631 0 : masm.jump(ool->rejoin());
10632 0 : }
10633 :
10634 : typedef JSObject* (*BindVarFn)(JSContext*, HandleObject);
10635 0 : static const VMFunction BindVarInfo = FunctionInfo<BindVarFn>(jit::BindVar, "BindVar");
10636 :
10637 : void
10638 0 : CodeGenerator::visitCallBindVar(LCallBindVar* lir)
10639 : {
10640 0 : pushArg(ToRegister(lir->environmentChain()));
10641 0 : callVM(BindVarInfo, lir);
10642 0 : }
10643 :
10644 : typedef bool (*GetPropertyFn)(JSContext*, HandleValue, HandlePropertyName, MutableHandleValue);
10645 1 : static const VMFunction GetPropertyInfo = FunctionInfo<GetPropertyFn>(GetProperty, "GetProperty");
10646 :
10647 : void
10648 9 : CodeGenerator::visitCallGetProperty(LCallGetProperty* lir)
10649 : {
10650 0 : pushArg(ImmGCPtr(lir->mir()->name()));
10651 18 : pushArg(ToValue(lir, LCallGetProperty::Value));
10652 :
10653 0 : callVM(GetPropertyInfo, lir);
10654 9 : }
10655 :
10656 : typedef bool (*GetOrCallElementFn)(JSContext*, MutableHandleValue, HandleValue, MutableHandleValue);
10657 1 : static const VMFunction GetElementInfo =
10658 0 : FunctionInfo<GetOrCallElementFn>(js::GetElement, "GetElement");
10659 0 : static const VMFunction CallElementInfo =
10660 3 : FunctionInfo<GetOrCallElementFn>(js::CallElement, "CallElement");
10661 :
10662 : void
10663 0 : CodeGenerator::visitCallGetElement(LCallGetElement* lir)
10664 : {
10665 0 : pushArg(ToValue(lir, LCallGetElement::RhsInput));
10666 0 : pushArg(ToValue(lir, LCallGetElement::LhsInput));
10667 :
10668 0 : JSOp op = JSOp(*lir->mir()->resumePoint()->pc());
10669 :
10670 0 : if (op == JSOP_GETELEM) {
10671 0 : callVM(GetElementInfo, lir);
10672 : } else {
10673 0 : MOZ_ASSERT(op == JSOP_CALLELEM);
10674 0 : callVM(CallElementInfo, lir);
10675 : }
10676 0 : }
10677 :
10678 : void
10679 0 : CodeGenerator::visitCallSetElement(LCallSetElement* lir)
10680 : {
10681 0 : Register obj = ToRegister(lir->getOperand(0));
10682 0 : pushArg(Imm32(lir->mir()->strict()));
10683 0 : pushArg(TypedOrValueRegister(MIRType::Object, AnyRegister(obj)));
10684 0 : pushArg(ToValue(lir, LCallSetElement::Value));
10685 0 : pushArg(ToValue(lir, LCallSetElement::Index));
10686 0 : pushArg(obj);
10687 0 : callVM(SetObjectElementInfo, lir);
10688 0 : }
10689 :
10690 : typedef bool (*InitElementArrayFn)(JSContext*, jsbytecode*, HandleObject, uint32_t, HandleValue);
10691 0 : static const VMFunction InitElementArrayInfo =
10692 0 : FunctionInfo<InitElementArrayFn>(js::InitElementArray, "InitElementArray");
10693 :
10694 : void
10695 0 : CodeGenerator::visitCallInitElementArray(LCallInitElementArray* lir)
10696 : {
10697 0 : pushArg(ToValue(lir, LCallInitElementArray::Value));
10698 0 : if (lir->index()->isConstant())
10699 0 : pushArg(Imm32(ToInt32(lir->index())));
10700 : else
10701 0 : pushArg(ToRegister(lir->index()));
10702 0 : pushArg(ToRegister(lir->object()));
10703 0 : pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
10704 0 : callVM(InitElementArrayInfo, lir);
10705 0 : }
10706 :
10707 : void
10708 0 : CodeGenerator::visitLoadFixedSlotV(LLoadFixedSlotV* ins)
10709 : {
10710 0 : const Register obj = ToRegister(ins->getOperand(0));
10711 86 : size_t slot = ins->mir()->slot();
10712 43 : ValueOperand result = ToOutValue(ins);
10713 :
10714 86 : masm.loadValue(Address(obj, NativeObject::getFixedSlotOffset(slot)), result);
10715 0 : }
10716 :
10717 : void
10718 89 : CodeGenerator::visitLoadFixedSlotT(LLoadFixedSlotT* ins)
10719 : {
10720 0 : const Register obj = ToRegister(ins->getOperand(0));
10721 178 : size_t slot = ins->mir()->slot();
10722 267 : AnyRegister result = ToAnyRegister(ins->getDef(0));
10723 0 : MIRType type = ins->mir()->type();
10724 :
10725 0 : masm.loadUnboxedValue(Address(obj, NativeObject::getFixedSlotOffset(slot)), type, result);
10726 0 : }
10727 :
10728 : void
10729 33 : CodeGenerator::visitLoadFixedSlotAndUnbox(LLoadFixedSlotAndUnbox* ins)
10730 : {
10731 0 : const MLoadFixedSlotAndUnbox* mir = ins->mir();
10732 33 : MIRType type = mir->type();
10733 99 : const Register input = ToRegister(ins->getOperand(0));
10734 0 : AnyRegister result = ToAnyRegister(ins->output());
10735 33 : size_t slot = mir->slot();
10736 :
10737 0 : Address address(input, NativeObject::getFixedSlotOffset(slot));
10738 0 : Label bail;
10739 0 : if (type == MIRType::Double) {
10740 0 : MOZ_ASSERT(result.isFloat());
10741 0 : masm.ensureDouble(address, result.fpu(), &bail);
10742 0 : if (mir->fallible())
10743 0 : bailoutFrom(&bail, ins->snapshot());
10744 0 : return;
10745 : }
10746 0 : if (mir->fallible()) {
10747 0 : switch (type) {
10748 : case MIRType::Int32:
10749 0 : masm.branchTestInt32(Assembler::NotEqual, address, &bail);
10750 : break;
10751 : case MIRType::Boolean:
10752 0 : masm.branchTestBoolean(Assembler::NotEqual, address, &bail);
10753 : break;
10754 : default:
10755 0 : MOZ_CRASH("Given MIRType cannot be unboxed.");
10756 : }
10757 0 : bailoutFrom(&bail, ins->snapshot());
10758 : }
10759 33 : masm.loadUnboxedValue(address, type, result);
10760 : }
10761 :
10762 : void
10763 19 : CodeGenerator::visitStoreFixedSlotV(LStoreFixedSlotV* ins)
10764 : {
10765 57 : const Register obj = ToRegister(ins->getOperand(0));
10766 38 : size_t slot = ins->mir()->slot();
10767 :
10768 0 : const ValueOperand value = ToValue(ins, LStoreFixedSlotV::Value);
10769 :
10770 0 : Address address(obj, NativeObject::getFixedSlotOffset(slot));
10771 0 : if (ins->mir()->needsBarrier())
10772 19 : emitPreBarrier(address);
10773 :
10774 19 : masm.storeValue(value, address);
10775 0 : }
10776 :
10777 : void
10778 250 : CodeGenerator::visitStoreFixedSlotT(LStoreFixedSlotT* ins)
10779 : {
10780 0 : const Register obj = ToRegister(ins->getOperand(0));
10781 500 : size_t slot = ins->mir()->slot();
10782 :
10783 0 : const LAllocation* value = ins->value();
10784 750 : MIRType valueType = ins->mir()->value()->type();
10785 :
10786 0 : Address address(obj, NativeObject::getFixedSlotOffset(slot));
10787 500 : if (ins->mir()->needsBarrier())
10788 0 : emitPreBarrier(address);
10789 :
10790 250 : if (valueType == MIRType::ObjectOrNull) {
10791 0 : Register nvalue = ToRegister(value);
10792 0 : masm.storeObjectOrNull(nvalue, address);
10793 : } else {
10794 250 : ConstantOrRegister nvalue = value->isConstant()
10795 0 : ? ConstantOrRegister(value->toConstant()->toJSValue())
10796 0 : : TypedOrValueRegister(valueType, ToAnyRegister(value));
10797 0 : masm.storeConstantOrRegister(nvalue, address);
10798 : }
10799 0 : }
10800 :
10801 : void
10802 0 : CodeGenerator::visitGetNameCache(LGetNameCache* ins)
10803 : {
10804 0 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10805 106 : Register envChain = ToRegister(ins->envObj());
10806 53 : ValueOperand output = ToOutValue(ins);
10807 0 : Register temp = ToRegister(ins->temp());
10808 :
10809 0 : IonGetNameIC ic(liveRegs, envChain, output, temp);
10810 0 : addIC(ins, allocateIC(ic));
10811 0 : }
10812 :
10813 : void
10814 0 : CodeGenerator::addGetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs,
10815 : TypedOrValueRegister value, const ConstantOrRegister& id,
10816 : TypedOrValueRegister output, Register maybeTemp,
10817 : GetPropertyResultFlags resultFlags)
10818 : {
10819 0 : CacheKind kind = CacheKind::GetElem;
10820 441 : if (id.constant() && id.value().isString()) {
10821 152 : JSString* idString = id.value().toString();
10822 : uint32_t dummy;
10823 304 : if (idString->isAtom() && !idString->asAtom().isIndex(&dummy))
10824 0 : kind = CacheKind::GetProp;
10825 : }
10826 0 : IonGetPropertyIC cache(kind, liveRegs, value, id, output, maybeTemp, resultFlags);
10827 271 : addIC(ins, allocateIC(cache));
10828 0 : }
10829 :
10830 : void
10831 0 : CodeGenerator::addSetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs, Register objReg,
10832 : Register temp, FloatRegister tempDouble,
10833 : FloatRegister tempF32, const ConstantOrRegister& id,
10834 : const ConstantOrRegister& value,
10835 : bool strict, bool needsPostBarrier, bool needsTypeBarrier,
10836 : bool guardHoles)
10837 : {
10838 42 : CacheKind kind = CacheKind::SetElem;
10839 65 : if (id.constant() && id.value().isString()) {
10840 22 : JSString* idString = id.value().toString();
10841 : uint32_t dummy;
10842 44 : if (idString->isAtom() && !idString->asAtom().isIndex(&dummy))
10843 0 : kind = CacheKind::SetProp;
10844 : }
10845 : IonSetPropertyIC cache(kind, liveRegs, objReg, temp, tempDouble, tempF32,
10846 126 : id, value, strict, needsPostBarrier, needsTypeBarrier, guardHoles);
10847 0 : addIC(ins, allocateIC(cache));
10848 0 : }
10849 :
10850 : ConstantOrRegister
10851 0 : CodeGenerator::toConstantOrRegister(LInstruction* lir, size_t n, MIRType type)
10852 : {
10853 0 : if (type == MIRType::Value)
10854 60 : return TypedOrValueRegister(ToValue(lir, n));
10855 :
10856 0 : const LAllocation* value = lir->getOperand(n);
10857 632 : if (value->isConstant())
10858 0 : return ConstantOrRegister(value->toConstant()->toJSValue());
10859 :
10860 1290 : return TypedOrValueRegister(type, ToAnyRegister(value));
10861 : }
10862 :
10863 : static GetPropertyResultFlags
10864 271 : IonGetPropertyICFlags(const MGetPropertyCache* mir)
10865 : {
10866 271 : GetPropertyResultFlags flags = GetPropertyResultFlags::None;
10867 271 : if (mir->monitoredResult())
10868 : flags |= GetPropertyResultFlags::Monitored;
10869 :
10870 271 : if (mir->type() == MIRType::Value) {
10871 0 : if (TemporaryTypeSet* types = mir->resultTypeSet()) {
10872 0 : if (types->hasType(TypeSet::UndefinedType()))
10873 : flags |= GetPropertyResultFlags::AllowUndefined;
10874 0 : if (types->hasType(TypeSet::Int32Type()))
10875 : flags |= GetPropertyResultFlags::AllowInt32;
10876 0 : if (types->hasType(TypeSet::DoubleType()))
10877 : flags |= GetPropertyResultFlags::AllowDouble;
10878 : } else {
10879 : flags |= GetPropertyResultFlags::AllowUndefined
10880 : | GetPropertyResultFlags::AllowInt32
10881 0 : | GetPropertyResultFlags::AllowDouble;
10882 : }
10883 5 : } else if (mir->type() == MIRType::Int32) {
10884 : flags |= GetPropertyResultFlags::AllowInt32;
10885 4 : } else if (mir->type() == MIRType::Double) {
10886 0 : flags |= GetPropertyResultFlags::AllowInt32 | GetPropertyResultFlags::AllowDouble;
10887 : }
10888 :
10889 271 : return flags;
10890 : }
10891 :
10892 : void
10893 266 : CodeGenerator::visitGetPropertyCacheV(LGetPropertyCacheV* ins)
10894 : {
10895 532 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10896 : TypedOrValueRegister value =
10897 798 : toConstantOrRegister(ins, LGetPropertyCacheV::Value, ins->mir()->value()->type()).reg();
10898 0 : ConstantOrRegister id = toConstantOrRegister(ins, LGetPropertyCacheV::Id, ins->mir()->idval()->type());
10899 532 : TypedOrValueRegister output(ToOutValue(ins));
10900 0 : Register maybeTemp = ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp());
10901 :
10902 0 : addGetPropertyCache(ins, liveRegs, value, id, output, maybeTemp,
10903 0 : IonGetPropertyICFlags(ins->mir()));
10904 0 : }
10905 :
10906 : void
10907 0 : CodeGenerator::visitGetPropertyCacheT(LGetPropertyCacheT* ins)
10908 : {
10909 0 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10910 : TypedOrValueRegister value =
10911 15 : toConstantOrRegister(ins, LGetPropertyCacheV::Value, ins->mir()->value()->type()).reg();
10912 0 : ConstantOrRegister id = toConstantOrRegister(ins, LGetPropertyCacheT::Id, ins->mir()->idval()->type());
10913 20 : TypedOrValueRegister output(ins->mir()->type(), ToAnyRegister(ins->getDef(0)));
10914 0 : Register maybeTemp = ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp());
10915 :
10916 0 : addGetPropertyCache(ins, liveRegs, value, id, output, maybeTemp,
10917 0 : IonGetPropertyICFlags(ins->mir()));
10918 0 : }
10919 :
10920 : void
10921 0 : CodeGenerator::visitGetPropSuperCacheV(LGetPropSuperCacheV* ins)
10922 : {
10923 0 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10924 4 : Register obj = ToRegister(ins->obj());
10925 : TypedOrValueRegister receiver =
10926 0 : toConstantOrRegister(ins, LGetPropSuperCacheV::Receiver, ins->mir()->receiver()->type()).reg();
10927 6 : ConstantOrRegister id = toConstantOrRegister(ins, LGetPropSuperCacheV::Id, ins->mir()->idval()->type());
10928 0 : TypedOrValueRegister output(ToOutValue(ins));
10929 :
10930 2 : CacheKind kind = CacheKind::GetElemSuper;
10931 0 : if (id.constant() && id.value().isString()) {
10932 0 : JSString* idString = id.value().toString();
10933 : uint32_t dummy;
10934 4 : if (idString->isAtom() && !idString->asAtom().isIndex(&dummy))
10935 0 : kind = CacheKind::GetPropSuper;
10936 : }
10937 :
10938 4 : IonGetPropSuperIC cache(kind, liveRegs, obj, receiver, id, output);
10939 0 : addIC(ins, allocateIC(cache));
10940 0 : }
10941 :
10942 : void
10943 0 : CodeGenerator::visitBindNameCache(LBindNameCache* ins)
10944 : {
10945 0 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10946 0 : Register envChain = ToRegister(ins->environmentChain());
10947 0 : Register output = ToRegister(ins->output());
10948 0 : Register temp = ToRegister(ins->temp());
10949 :
10950 0 : IonBindNameIC ic(liveRegs, envChain, output, temp);
10951 0 : addIC(ins, allocateIC(ic));
10952 0 : }
10953 :
10954 : void
10955 0 : CodeGenerator::visitHasOwnCache(LHasOwnCache* ins)
10956 : {
10957 0 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
10958 : TypedOrValueRegister value =
10959 3 : toConstantOrRegister(ins, LHasOwnCache::Value, ins->mir()->value()->type()).reg();
10960 : TypedOrValueRegister id =
10961 3 : toConstantOrRegister(ins, LHasOwnCache::Id, ins->mir()->idval()->type()).reg();
10962 0 : Register output = ToRegister(ins->output());
10963 :
10964 0 : IonHasOwnIC cache(liveRegs, value, id, output);
10965 1 : addIC(ins, allocateIC(cache));
10966 0 : }
10967 :
10968 : typedef bool (*SetPropertyFn)(JSContext*, HandleObject,
10969 : HandlePropertyName, const HandleValue, bool, jsbytecode*);
10970 0 : static const VMFunction SetPropertyInfo = FunctionInfo<SetPropertyFn>(SetProperty, "SetProperty");
10971 :
10972 : void
10973 0 : CodeGenerator::visitCallSetProperty(LCallSetProperty* ins)
10974 : {
10975 0 : ConstantOrRegister value = TypedOrValueRegister(ToValue(ins, LCallSetProperty::Value));
10976 :
10977 0 : const Register objReg = ToRegister(ins->getOperand(0));
10978 :
10979 0 : pushArg(ImmPtr(ins->mir()->resumePoint()->pc()));
10980 0 : pushArg(Imm32(ins->mir()->strict()));
10981 :
10982 0 : pushArg(value);
10983 0 : pushArg(ImmGCPtr(ins->mir()->name()));
10984 0 : pushArg(objReg);
10985 :
10986 0 : callVM(SetPropertyInfo, ins);
10987 0 : }
10988 :
10989 : typedef bool (*DeletePropertyFn)(JSContext*, HandleValue, HandlePropertyName, bool*);
10990 1 : static const VMFunction DeletePropertyStrictInfo =
10991 1 : FunctionInfo<DeletePropertyFn>(DeletePropertyJit<true>, "DeletePropertyStrict");
10992 1 : static const VMFunction DeletePropertyNonStrictInfo =
10993 3 : FunctionInfo<DeletePropertyFn>(DeletePropertyJit<false>, "DeletePropertyNonStrict");
10994 :
10995 : void
10996 0 : CodeGenerator::visitCallDeleteProperty(LCallDeleteProperty* lir)
10997 : {
10998 0 : pushArg(ImmGCPtr(lir->mir()->name()));
10999 0 : pushArg(ToValue(lir, LCallDeleteProperty::Value));
11000 :
11001 0 : if (lir->mir()->strict())
11002 0 : callVM(DeletePropertyStrictInfo, lir);
11003 : else
11004 0 : callVM(DeletePropertyNonStrictInfo, lir);
11005 0 : }
11006 :
11007 : typedef bool (*DeleteElementFn)(JSContext*, HandleValue, HandleValue, bool*);
11008 1 : static const VMFunction DeleteElementStrictInfo =
11009 0 : FunctionInfo<DeleteElementFn>(DeleteElementJit<true>, "DeleteElementStrict");
11010 0 : static const VMFunction DeleteElementNonStrictInfo =
11011 3 : FunctionInfo<DeleteElementFn>(DeleteElementJit<false>, "DeleteElementNonStrict");
11012 :
11013 : void
11014 0 : CodeGenerator::visitCallDeleteElement(LCallDeleteElement* lir)
11015 : {
11016 0 : pushArg(ToValue(lir, LCallDeleteElement::Index));
11017 0 : pushArg(ToValue(lir, LCallDeleteElement::Value));
11018 :
11019 0 : if (lir->mir()->strict())
11020 0 : callVM(DeleteElementStrictInfo, lir);
11021 : else
11022 0 : callVM(DeleteElementNonStrictInfo, lir);
11023 0 : }
11024 :
11025 : void
11026 42 : CodeGenerator::visitSetPropertyCache(LSetPropertyCache* ins)
11027 : {
11028 0 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
11029 126 : Register objReg = ToRegister(ins->getOperand(0));
11030 84 : Register temp = ToRegister(ins->temp());
11031 0 : FloatRegister tempDouble = ToTempFloatRegisterOrInvalid(ins->tempDouble());
11032 42 : FloatRegister tempF32 = ToTempFloatRegisterOrInvalid(ins->tempFloat32());
11033 :
11034 : ConstantOrRegister id =
11035 0 : toConstantOrRegister(ins, LSetPropertyCache::Id, ins->mir()->idval()->type());
11036 : ConstantOrRegister value =
11037 0 : toConstantOrRegister(ins, LSetPropertyCache::Value, ins->mir()->value()->type());
11038 :
11039 210 : addSetPropertyCache(ins, liveRegs, objReg, temp, tempDouble, tempF32,
11040 0 : id, value, ins->mir()->strict(), ins->mir()->needsPostBarrier(),
11041 294 : ins->mir()->needsTypeBarrier(), ins->mir()->guardHoles());
11042 0 : }
11043 :
11044 : typedef bool (*ThrowFn)(JSContext*, HandleValue);
11045 0 : static const VMFunction ThrowInfoCodeGen = FunctionInfo<ThrowFn>(js::Throw, "Throw");
11046 :
11047 : void
11048 0 : CodeGenerator::visitThrow(LThrow* lir)
11049 : {
11050 0 : pushArg(ToValue(lir, LThrow::Value));
11051 0 : callVM(ThrowInfoCodeGen, lir);
11052 0 : }
11053 :
11054 : typedef bool (*BitNotFn)(JSContext*, HandleValue, int* p);
11055 0 : static const VMFunction BitNotInfo = FunctionInfo<BitNotFn>(BitNot, "BitNot");
11056 :
11057 : void
11058 0 : CodeGenerator::visitBitNotV(LBitNotV* lir)
11059 : {
11060 0 : pushArg(ToValue(lir, LBitNotV::Input));
11061 0 : callVM(BitNotInfo, lir);
11062 0 : }
11063 :
11064 : typedef bool (*BitopFn)(JSContext*, HandleValue, HandleValue, int* p);
11065 1 : static const VMFunction BitAndInfo = FunctionInfo<BitopFn>(BitAnd, "BitAnd");
11066 1 : static const VMFunction BitOrInfo = FunctionInfo<BitopFn>(BitOr, "BitOr");
11067 1 : static const VMFunction BitXorInfo = FunctionInfo<BitopFn>(BitXor, "BitXor");
11068 2 : static const VMFunction BitLhsInfo = FunctionInfo<BitopFn>(BitLsh, "BitLsh");
11069 2 : static const VMFunction BitRhsInfo = FunctionInfo<BitopFn>(BitRsh, "BitRsh");
11070 :
11071 : void
11072 0 : CodeGenerator::visitBitOpV(LBitOpV* lir)
11073 : {
11074 0 : pushArg(ToValue(lir, LBitOpV::RhsInput));
11075 0 : pushArg(ToValue(lir, LBitOpV::LhsInput));
11076 :
11077 0 : switch (lir->jsop()) {
11078 : case JSOP_BITAND:
11079 0 : callVM(BitAndInfo, lir);
11080 0 : break;
11081 : case JSOP_BITOR:
11082 0 : callVM(BitOrInfo, lir);
11083 0 : break;
11084 : case JSOP_BITXOR:
11085 0 : callVM(BitXorInfo, lir);
11086 0 : break;
11087 : case JSOP_LSH:
11088 0 : callVM(BitLhsInfo, lir);
11089 0 : break;
11090 : case JSOP_RSH:
11091 0 : callVM(BitRhsInfo, lir);
11092 0 : break;
11093 : default:
11094 0 : MOZ_CRASH("unexpected bitop");
11095 : }
11096 0 : }
11097 :
11098 : class OutOfLineTypeOfV : public OutOfLineCodeBase<CodeGenerator>
11099 : {
11100 : LTypeOfV* ins_;
11101 :
11102 : public:
11103 : explicit OutOfLineTypeOfV(LTypeOfV* ins)
11104 12 : : ins_(ins)
11105 : { }
11106 :
11107 6 : void accept(CodeGenerator* codegen) override {
11108 6 : codegen->visitOutOfLineTypeOfV(this);
11109 0 : }
11110 : LTypeOfV* ins() const {
11111 : return ins_;
11112 : }
11113 : };
11114 :
11115 : void
11116 6 : CodeGenerator::visitTypeOfV(LTypeOfV* lir)
11117 : {
11118 6 : const ValueOperand value = ToValue(lir, LTypeOfV::Input);
11119 12 : Register output = ToRegister(lir->output());
11120 6 : Register tag = masm.extractTag(value, output);
11121 :
11122 12 : const JSAtomState& names = gen->runtime->names();
11123 0 : Label done;
11124 :
11125 0 : MDefinition* input = lir->mir()->input();
11126 :
11127 0 : bool testObject = input->mightBeType(MIRType::Object);
11128 0 : bool testNumber = input->mightBeType(MIRType::Int32) || input->mightBeType(MIRType::Double);
11129 6 : bool testBoolean = input->mightBeType(MIRType::Boolean);
11130 0 : bool testUndefined = input->mightBeType(MIRType::Undefined);
11131 6 : bool testNull = input->mightBeType(MIRType::Null);
11132 0 : bool testString = input->mightBeType(MIRType::String);
11133 0 : bool testSymbol = input->mightBeType(MIRType::Symbol);
11134 :
11135 0 : unsigned numTests = unsigned(testObject) + unsigned(testNumber) + unsigned(testBoolean) +
11136 0 : unsigned(testUndefined) + unsigned(testNull) + unsigned(testString) + unsigned(testSymbol);
11137 :
11138 0 : MOZ_ASSERT_IF(!input->emptyResultTypeSet(), numTests > 0);
11139 :
11140 0 : OutOfLineTypeOfV* ool = nullptr;
11141 0 : if (testObject) {
11142 12 : if (lir->mir()->inputMaybeCallableOrEmulatesUndefined()) {
11143 : // The input may be a callable object (result is "function") or may
11144 : // emulate undefined (result is "undefined"). Use an OOL path.
11145 0 : ool = new(alloc()) OutOfLineTypeOfV(lir);
11146 0 : addOutOfLineCode(ool, lir->mir());
11147 :
11148 6 : if (numTests > 1)
11149 12 : masm.branchTestObject(Assembler::Equal, tag, ool->entry());
11150 : else
11151 0 : masm.jump(ool->entry());
11152 : } else {
11153 : // Input is not callable and does not emulate undefined, so if
11154 : // it's an object the result is always "object".
11155 0 : Label notObject;
11156 0 : if (numTests > 1)
11157 0 : masm.branchTestObject(Assembler::NotEqual, tag, ¬Object);
11158 0 : masm.movePtr(ImmGCPtr(names.object), output);
11159 0 : if (numTests > 1)
11160 0 : masm.jump(&done);
11161 0 : masm.bind(¬Object);
11162 : }
11163 0 : numTests--;
11164 : }
11165 :
11166 0 : if (testNumber) {
11167 8 : Label notNumber;
11168 0 : if (numTests > 1)
11169 4 : masm.branchTestNumber(Assembler::NotEqual, tag, ¬Number);
11170 12 : masm.movePtr(ImmGCPtr(names.number), output);
11171 0 : if (numTests > 1)
11172 0 : masm.jump(&done);
11173 0 : masm.bind(¬Number);
11174 0 : numTests--;
11175 : }
11176 :
11177 0 : if (testUndefined) {
11178 0 : Label notUndefined;
11179 0 : if (numTests > 1)
11180 0 : masm.branchTestUndefined(Assembler::NotEqual, tag, ¬Undefined);
11181 0 : masm.movePtr(ImmGCPtr(names.undefined), output);
11182 0 : if (numTests > 1)
11183 0 : masm.jump(&done);
11184 0 : masm.bind(¬Undefined);
11185 0 : numTests--;
11186 : }
11187 :
11188 0 : if (testNull) {
11189 0 : Label notNull;
11190 0 : if (numTests > 1)
11191 0 : masm.branchTestNull(Assembler::NotEqual, tag, ¬Null);
11192 0 : masm.movePtr(ImmGCPtr(names.object), output);
11193 0 : if (numTests > 1)
11194 0 : masm.jump(&done);
11195 0 : masm.bind(¬Null);
11196 0 : numTests--;
11197 : }
11198 :
11199 0 : if (testBoolean) {
11200 0 : Label notBoolean;
11201 0 : if (numTests > 1)
11202 3 : masm.branchTestBoolean(Assembler::NotEqual, tag, ¬Boolean);
11203 12 : masm.movePtr(ImmGCPtr(names.boolean), output);
11204 0 : if (numTests > 1)
11205 0 : masm.jump(&done);
11206 0 : masm.bind(¬Boolean);
11207 0 : numTests--;
11208 : }
11209 :
11210 0 : if (testString) {
11211 0 : Label notString;
11212 0 : if (numTests > 1)
11213 0 : masm.branchTestString(Assembler::NotEqual, tag, ¬String);
11214 15 : masm.movePtr(ImmGCPtr(names.string), output);
11215 0 : if (numTests > 1)
11216 0 : masm.jump(&done);
11217 0 : masm.bind(¬String);
11218 0 : numTests--;
11219 : }
11220 :
11221 0 : if (testSymbol) {
11222 0 : Label notSymbol;
11223 0 : if (numTests > 1)
11224 0 : masm.branchTestSymbol(Assembler::NotEqual, tag, ¬Symbol);
11225 0 : masm.movePtr(ImmGCPtr(names.symbol), output);
11226 0 : if (numTests > 1)
11227 0 : masm.jump(&done);
11228 0 : masm.bind(¬Symbol);
11229 0 : numTests--;
11230 : }
11231 :
11232 1 : MOZ_ASSERT(numTests == 0);
11233 :
11234 0 : masm.bind(&done);
11235 6 : if (ool)
11236 12 : masm.bind(ool->rejoin());
11237 0 : }
11238 :
11239 : void
11240 0 : CodeGenerator::visitOutOfLineTypeOfV(OutOfLineTypeOfV* ool)
11241 : {
11242 0 : LTypeOfV* ins = ool->ins();
11243 12 : const JSAtomState& names = gen->runtime->names();
11244 :
11245 0 : ValueOperand input = ToValue(ins, LTypeOfV::Input);
11246 12 : Register temp = ToTempUnboxRegister(ins->tempToUnbox());
11247 0 : Register output = ToRegister(ins->output());
11248 :
11249 6 : Register obj = masm.extractObject(input, temp);
11250 :
11251 0 : Label slowCheck, isObject, isCallable, isUndefined, done;
11252 0 : masm.typeOfObject(obj, output, &slowCheck, &isObject, &isCallable, &isUndefined);
11253 :
11254 0 : masm.bind(&isCallable);
11255 18 : masm.movePtr(ImmGCPtr(names.function), output);
11256 0 : masm.jump(ool->rejoin());
11257 :
11258 6 : masm.bind(&isUndefined);
11259 0 : masm.movePtr(ImmGCPtr(names.undefined), output);
11260 0 : masm.jump(ool->rejoin());
11261 :
11262 6 : masm.bind(&isObject);
11263 0 : masm.movePtr(ImmGCPtr(names.object), output);
11264 0 : masm.jump(ool->rejoin());
11265 :
11266 6 : masm.bind(&slowCheck);
11267 :
11268 0 : saveVolatile(output);
11269 0 : masm.setupUnalignedABICall(output);
11270 12 : masm.passABIArg(obj);
11271 0 : masm.movePtr(ImmPtr(gen->runtime), output);
11272 12 : masm.passABIArg(output);
11273 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, jit::TypeOfObject));
11274 0 : masm.storeCallPointerResult(output);
11275 0 : restoreVolatile(output);
11276 :
11277 0 : masm.jump(ool->rejoin());
11278 0 : }
11279 :
11280 : typedef JSObject* (*ToAsyncFn)(JSContext*, HandleFunction);
11281 2 : static const VMFunction ToAsyncInfo = FunctionInfo<ToAsyncFn>(js::WrapAsyncFunction, "ToAsync");
11282 :
11283 : void
11284 0 : CodeGenerator::visitToAsync(LToAsync* lir)
11285 : {
11286 0 : pushArg(ToRegister(lir->unwrapped()));
11287 0 : callVM(ToAsyncInfo, lir);
11288 0 : }
11289 :
11290 : typedef JSObject* (*ToAsyncGenFn)(JSContext*, HandleFunction);
11291 0 : static const VMFunction ToAsyncGenInfo =
11292 0 : FunctionInfo<ToAsyncGenFn>(js::WrapAsyncGenerator, "ToAsyncGen");
11293 :
11294 : void
11295 0 : CodeGenerator::visitToAsyncGen(LToAsyncGen* lir)
11296 : {
11297 0 : pushArg(ToRegister(lir->unwrapped()));
11298 0 : callVM(ToAsyncGenInfo, lir);
11299 0 : }
11300 :
11301 : typedef JSObject* (*ToAsyncIterFn)(JSContext*, HandleObject, HandleValue);
11302 0 : static const VMFunction ToAsyncIterInfo =
11303 0 : FunctionInfo<ToAsyncIterFn>(js::CreateAsyncFromSyncIterator, "ToAsyncIter");
11304 :
11305 : void
11306 0 : CodeGenerator::visitToAsyncIter(LToAsyncIter* lir)
11307 : {
11308 0 : pushArg(ToValue(lir, LToAsyncIter::NextMethodIndex));
11309 0 : pushArg(ToRegister(lir->iterator()));
11310 0 : callVM(ToAsyncIterInfo, lir);
11311 0 : }
11312 :
11313 : typedef bool (*ToIdFn)(JSContext*, HandleValue, MutableHandleValue);
11314 1 : static const VMFunction ToIdInfo = FunctionInfo<ToIdFn>(ToIdOperation, "ToIdOperation");
11315 :
11316 : void
11317 0 : CodeGenerator::visitToIdV(LToIdV* lir)
11318 : {
11319 0 : Label notInt32;
11320 0 : FloatRegister temp = ToFloatRegister(lir->tempFloat());
11321 0 : const ValueOperand out = ToOutValue(lir);
11322 0 : ValueOperand input = ToValue(lir, LToIdV::Input);
11323 :
11324 0 : OutOfLineCode* ool = oolCallVM(ToIdInfo, lir,
11325 0 : ArgList(ToValue(lir, LToIdV::Input)),
11326 0 : StoreValueTo(out));
11327 :
11328 0 : Register tag = masm.extractTag(input, out.scratchReg());
11329 :
11330 0 : masm.branchTestInt32(Assembler::NotEqual, tag, ¬Int32);
11331 0 : masm.moveValue(input, out);
11332 0 : masm.jump(ool->rejoin());
11333 :
11334 0 : masm.bind(¬Int32);
11335 0 : masm.branchTestDouble(Assembler::NotEqual, tag, ool->entry());
11336 0 : masm.unboxDouble(input, temp);
11337 0 : masm.convertDoubleToInt32(temp, out.scratchReg(), ool->entry(), true);
11338 0 : masm.tagValue(JSVAL_TYPE_INT32, out.scratchReg(), out);
11339 :
11340 0 : masm.bind(ool->rejoin());
11341 0 : }
11342 :
11343 : template<typename T>
11344 : void
11345 0 : CodeGenerator::emitLoadElementT(LLoadElementT* lir, const T& source)
11346 : {
11347 : if (LIRGenerator::allowTypedElementHoleCheck()) {
11348 : if (lir->mir()->needsHoleCheck()) {
11349 : Label bail;
11350 : masm.branchTestMagic(Assembler::Equal, source, &bail);
11351 : bailoutFrom(&bail, lir->snapshot());
11352 : }
11353 : } else {
11354 0 : MOZ_ASSERT(!lir->mir()->needsHoleCheck());
11355 : }
11356 :
11357 0 : AnyRegister output = ToAnyRegister(lir->output());
11358 0 : if (lir->mir()->loadDoubles())
11359 0 : masm.loadDouble(source, output.fpu());
11360 : else
11361 0 : masm.loadUnboxedValue(source, lir->mir()->type(), output);
11362 0 : }
11363 :
11364 : void
11365 0 : CodeGenerator::visitLoadElementT(LLoadElementT* lir)
11366 : {
11367 0 : Register elements = ToRegister(lir->elements());
11368 0 : const LAllocation* index = lir->index();
11369 0 : if (index->isConstant()) {
11370 0 : int32_t offset = ToInt32(index) * sizeof(js::Value) + lir->mir()->offsetAdjustment();
11371 0 : emitLoadElementT(lir, Address(elements, offset));
11372 : } else {
11373 0 : emitLoadElementT(lir, BaseIndex(elements, ToRegister(index), TimesEight,
11374 0 : lir->mir()->offsetAdjustment()));
11375 : }
11376 0 : }
11377 :
11378 : void
11379 0 : CodeGenerator::visitLoadElementV(LLoadElementV* load)
11380 : {
11381 0 : Register elements = ToRegister(load->elements());
11382 33 : const ValueOperand out = ToOutValue(load);
11383 :
11384 0 : if (load->index()->isConstant()) {
11385 0 : NativeObject::elementsSizeMustNotOverflow();
11386 0 : int32_t offset = ToInt32(load->index()) * sizeof(Value) + load->mir()->offsetAdjustment();
11387 0 : masm.loadValue(Address(elements, offset), out);
11388 : } else {
11389 0 : masm.loadValue(BaseObjectElementIndex(elements, ToRegister(load->index()),
11390 0 : load->mir()->offsetAdjustment()), out);
11391 : }
11392 :
11393 66 : if (load->mir()->needsHoleCheck()) {
11394 0 : Label testMagic;
11395 0 : masm.branchTestMagic(Assembler::Equal, out, &testMagic);
11396 0 : bailoutFrom(&testMagic, load->snapshot());
11397 : }
11398 0 : }
11399 :
11400 : void
11401 0 : CodeGenerator::visitLoadElementHole(LLoadElementHole* lir)
11402 : {
11403 0 : Register elements = ToRegister(lir->elements());
11404 0 : Register index = ToRegister(lir->index());
11405 0 : Register initLength = ToRegister(lir->initLength());
11406 0 : const ValueOperand out = ToOutValue(lir);
11407 :
11408 0 : const MLoadElementHole* mir = lir->mir();
11409 :
11410 : // If the index is out of bounds, load |undefined|. Otherwise, load the
11411 : // value.
11412 0 : Label outOfBounds, done;
11413 0 : masm.spectreBoundsCheck32(index, initLength, out.scratchReg(), &outOfBounds);
11414 :
11415 0 : masm.loadValue(BaseObjectElementIndex(elements, index), out);
11416 :
11417 : // If a hole check is needed, and the value wasn't a hole, we're done.
11418 : // Otherwise, we'll load undefined.
11419 0 : if (lir->mir()->needsHoleCheck()) {
11420 0 : masm.branchTestMagic(Assembler::NotEqual, out, &done);
11421 0 : masm.moveValue(UndefinedValue(), out);
11422 : }
11423 0 : masm.jump(&done);
11424 :
11425 0 : masm.bind(&outOfBounds);
11426 0 : if (mir->needsNegativeIntCheck()) {
11427 0 : Label negative;
11428 0 : masm.branch32(Assembler::LessThan, index, Imm32(0), &negative);
11429 0 : bailoutFrom(&negative, lir->snapshot());
11430 : }
11431 0 : masm.moveValue(UndefinedValue(), out);
11432 :
11433 0 : masm.bind(&done);
11434 0 : }
11435 :
11436 : void
11437 0 : CodeGenerator::visitLoadUnboxedPointerV(LLoadUnboxedPointerV* lir)
11438 : {
11439 0 : Register elements = ToRegister(lir->elements());
11440 0 : const ValueOperand out = ToOutValue(lir);
11441 :
11442 0 : if (lir->index()->isConstant()) {
11443 0 : int32_t offset = ToInt32(lir->index()) * sizeof(uintptr_t) + lir->mir()->offsetAdjustment();
11444 0 : masm.loadPtr(Address(elements, offset), out.scratchReg());
11445 : } else {
11446 0 : masm.loadPtr(BaseIndex(elements, ToRegister(lir->index()), ScalePointer,
11447 0 : lir->mir()->offsetAdjustment()), out.scratchReg());
11448 : }
11449 :
11450 0 : Label notNull, done;
11451 0 : masm.branchPtr(Assembler::NotEqual, out.scratchReg(), ImmWord(0), ¬Null);
11452 :
11453 0 : masm.moveValue(NullValue(), out);
11454 0 : masm.jump(&done);
11455 :
11456 0 : masm.bind(¬Null);
11457 0 : masm.tagValue(JSVAL_TYPE_OBJECT, out.scratchReg(), out);
11458 :
11459 0 : masm.bind(&done);
11460 0 : }
11461 :
11462 : void
11463 4 : CodeGenerator::visitLoadUnboxedPointerT(LLoadUnboxedPointerT* lir)
11464 : {
11465 0 : Register elements = ToRegister(lir->elements());
11466 4 : const LAllocation* index = lir->index();
11467 8 : Register out = ToRegister(lir->output());
11468 :
11469 : bool bailOnNull;
11470 : int32_t offsetAdjustment;
11471 0 : if (lir->mir()->isLoadUnboxedObjectOrNull()) {
11472 0 : bailOnNull = lir->mir()->toLoadUnboxedObjectOrNull()->nullBehavior() ==
11473 : MLoadUnboxedObjectOrNull::BailOnNull;
11474 0 : offsetAdjustment = lir->mir()->toLoadUnboxedObjectOrNull()->offsetAdjustment();
11475 8 : } else if (lir->mir()->isLoadUnboxedString()) {
11476 0 : bailOnNull = false;
11477 0 : offsetAdjustment = lir->mir()->toLoadUnboxedString()->offsetAdjustment();
11478 : } else {
11479 0 : MOZ_CRASH();
11480 : }
11481 :
11482 0 : if (index->isConstant()) {
11483 8 : Address source(elements, ToInt32(index) * sizeof(uintptr_t) + offsetAdjustment);
11484 1 : masm.loadPtr(source, out);
11485 : } else {
11486 0 : BaseIndex source(elements, ToRegister(index), ScalePointer, offsetAdjustment);
11487 0 : masm.loadPtr(source, out);
11488 : }
11489 :
11490 4 : if (bailOnNull) {
11491 0 : Label bail;
11492 0 : masm.branchTestPtr(Assembler::Zero, out, out, &bail);
11493 0 : bailoutFrom(&bail, lir->snapshot());
11494 : }
11495 0 : }
11496 :
11497 : void
11498 0 : CodeGenerator::visitUnboxObjectOrNull(LUnboxObjectOrNull* lir)
11499 : {
11500 0 : Register obj = ToRegister(lir->input());
11501 :
11502 0 : if (lir->mir()->fallible()) {
11503 0 : Label bail;
11504 0 : masm.branchTestPtr(Assembler::Zero, obj, obj, &bail);
11505 0 : bailoutFrom(&bail, lir->snapshot());
11506 : }
11507 0 : }
11508 :
11509 : void
11510 0 : CodeGenerator::visitLoadUnboxedScalar(LLoadUnboxedScalar* lir)
11511 : {
11512 0 : Register elements = ToRegister(lir->elements());
11513 0 : Register temp = lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
11514 0 : AnyRegister out = ToAnyRegister(lir->output());
11515 :
11516 0 : const MLoadUnboxedScalar* mir = lir->mir();
11517 :
11518 0 : Scalar::Type readType = mir->readType();
11519 0 : unsigned numElems = mir->numElems();
11520 :
11521 0 : int width = Scalar::byteSize(mir->storageType());
11522 0 : bool canonicalizeDouble = mir->canonicalizeDoubles();
11523 :
11524 0 : Label fail;
11525 0 : if (lir->index()->isConstant()) {
11526 0 : Address source(elements, ToInt32(lir->index()) * width + mir->offsetAdjustment());
11527 0 : masm.loadFromTypedArray(readType, source, out, temp, &fail, canonicalizeDouble, numElems);
11528 : } else {
11529 : BaseIndex source(elements, ToRegister(lir->index()), ScaleFromElemWidth(width),
11530 0 : mir->offsetAdjustment());
11531 0 : masm.loadFromTypedArray(readType, source, out, temp, &fail, canonicalizeDouble, numElems);
11532 : }
11533 :
11534 0 : if (fail.used())
11535 0 : bailoutFrom(&fail, lir->snapshot());
11536 0 : }
11537 :
11538 : void
11539 0 : CodeGenerator::visitLoadTypedArrayElementHole(LLoadTypedArrayElementHole* lir)
11540 : {
11541 0 : Register object = ToRegister(lir->object());
11542 0 : const ValueOperand out = ToOutValue(lir);
11543 :
11544 : // Load the length.
11545 0 : Register scratch = out.scratchReg();
11546 0 : Register scratch2 = ToRegister(lir->temp());
11547 0 : Register index = ToRegister(lir->index());
11548 0 : masm.unboxInt32(Address(object, TypedArrayObject::lengthOffset()), scratch);
11549 :
11550 : // Load undefined if index >= length.
11551 0 : Label outOfBounds, done;
11552 0 : masm.spectreBoundsCheck32(index, scratch, scratch2, &outOfBounds);
11553 :
11554 : // Load the elements vector.
11555 0 : masm.loadPtr(Address(object, TypedArrayObject::dataOffset()), scratch);
11556 :
11557 0 : Scalar::Type arrayType = lir->mir()->arrayType();
11558 0 : int width = Scalar::byteSize(arrayType);
11559 0 : Label fail;
11560 0 : BaseIndex source(scratch, index, ScaleFromElemWidth(width));
11561 0 : masm.loadFromTypedArray(arrayType, source, out, lir->mir()->allowDouble(),
11562 0 : out.scratchReg(), &fail);
11563 0 : masm.jump(&done);
11564 :
11565 0 : masm.bind(&outOfBounds);
11566 0 : masm.moveValue(UndefinedValue(), out);
11567 :
11568 0 : if (fail.used())
11569 0 : bailoutFrom(&fail, lir->snapshot());
11570 :
11571 0 : masm.bind(&done);
11572 0 : }
11573 :
11574 : template <SwitchTableType tableType>
11575 : class OutOfLineSwitch : public OutOfLineCodeBase<CodeGenerator>
11576 : {
11577 : using LabelsVector = Vector<Label, 0, JitAllocPolicy>;
11578 : using CodeLabelsVector = Vector<CodeLabel, 0, JitAllocPolicy>;
11579 : LabelsVector labels_;
11580 : CodeLabelsVector codeLabels_;
11581 : CodeLabel start_;
11582 : bool isOutOfLine_;
11583 :
11584 0 : void accept(CodeGenerator* codegen) override {
11585 0 : codegen->visitOutOfLineSwitch(this);
11586 0 : }
11587 :
11588 : public:
11589 0 : explicit OutOfLineSwitch(TempAllocator& alloc)
11590 : : labels_(alloc),
11591 : codeLabels_(alloc),
11592 0 : isOutOfLine_(false)
11593 : {}
11594 :
11595 : CodeLabel* start() {
11596 0 : return &start_;
11597 : }
11598 :
11599 : CodeLabelsVector& codeLabels() {
11600 0 : return codeLabels_;
11601 : }
11602 : LabelsVector& labels() {
11603 0 : return labels_;
11604 : }
11605 :
11606 0 : void jumpToCodeEntries(MacroAssembler& masm, Register index, Register temp) {
11607 0 : Register base;
11608 : if (tableType == SwitchTableType::Inline) {
11609 : #if defined(JS_CODEGEN_ARM)
11610 : base = ::js::jit::pc;
11611 : #else
11612 : MOZ_CRASH("NYI: SwitchTableType::Inline");
11613 : #endif
11614 : } else {
11615 : #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
11616 : MOZ_CRASH("NYI: SwitchTableType::OutOfLine");
11617 : #else
11618 0 : masm.mov(start(), temp);
11619 0 : base = temp;
11620 : #endif
11621 : }
11622 0 : BaseIndex jumpTarget(base, index, ScalePointer);
11623 0 : masm.branchToComputedAddress(jumpTarget);
11624 0 : }
11625 :
11626 : // Register an entry in the switch table.
11627 0 : void addTableEntry(MacroAssembler& masm) {
11628 0 : if ((!isOutOfLine_ && tableType == SwitchTableType::Inline) ||
11629 0 : (isOutOfLine_ && tableType == SwitchTableType::OutOfLine))
11630 : {
11631 0 : CodeLabel cl;
11632 0 : masm.writeCodePointer(&cl);
11633 0 : masm.propagateOOM(codeLabels_.append(std::move(cl)));
11634 : }
11635 0 : }
11636 : // Register the code, to which the table will jump to.
11637 0 : void addCodeEntry(MacroAssembler& masm) {
11638 0 : Label entry;
11639 0 : masm.bind(&entry);
11640 0 : masm.propagateOOM(labels_.append(std::move(entry)));
11641 0 : }
11642 :
11643 : void setOutOfLine() {
11644 0 : isOutOfLine_ = true;
11645 : }
11646 : };
11647 :
11648 : template <SwitchTableType tableType>
11649 : void
11650 0 : CodeGenerator::visitOutOfLineSwitch(OutOfLineSwitch<tableType>* jumpTable)
11651 : {
11652 0 : jumpTable->setOutOfLine();
11653 : if (tableType == SwitchTableType::OutOfLine) {
11654 : #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
11655 : MOZ_CRASH("NYI: SwitchTableType::OutOfLine");
11656 : #elif defined(JS_CODEGEN_NONE)
11657 : MOZ_CRASH();
11658 : #else
11659 0 : masm.haltingAlign(sizeof(void*));
11660 0 : masm.bind(jumpTable->start());
11661 0 : masm.addCodeLabel(*jumpTable->start());
11662 : #endif
11663 : }
11664 :
11665 : // Add table entries if the table is inlined.
11666 0 : auto& labels = jumpTable->labels();
11667 0 : for (size_t i = 0, e = labels.length(); i < e; i++)
11668 0 : jumpTable->addTableEntry(masm);
11669 :
11670 0 : auto& codeLabels = jumpTable->codeLabels();
11671 0 : for (size_t i = 0, e = codeLabels.length(); i < e; i++) {
11672 : // The entries of the jump table need to be absolute addresses and thus
11673 : // must be patched after codegen is finished.
11674 0 : auto& cl = codeLabels[i];
11675 0 : cl.target()->bind(labels[i].offset());
11676 0 : masm.addCodeLabel(cl);
11677 : }
11678 0 : }
11679 :
11680 : template void CodeGenerator::visitOutOfLineSwitch(OutOfLineSwitch<SwitchTableType::Inline>* jumpTable);
11681 : template void CodeGenerator::visitOutOfLineSwitch(OutOfLineSwitch<SwitchTableType::OutOfLine>* jumpTable);
11682 :
11683 : void
11684 0 : CodeGenerator::visitLoadElementFromStateV(LLoadElementFromStateV* lir)
11685 : {
11686 0 : Register index = ToRegister(lir->index());
11687 0 : Register temp0 = ToRegister(lir->temp0());
11688 : #ifdef JS_NUNBOX32
11689 : Register temp1 = ToRegister(lir->temp1());
11690 : #endif
11691 0 : FloatRegister tempD = ToFloatRegister(lir->tempD());
11692 0 : ValueOperand out = ToOutValue(lir);
11693 :
11694 : // For each element, load it and box it.
11695 0 : MArgumentState* array = lir->array()->toArgumentState();
11696 0 : Label join;
11697 :
11698 : // Jump to the code which is loading the element, based on its index.
11699 : #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
11700 : auto* jumpTable = new (alloc()) OutOfLineSwitch<SwitchTableType::Inline>(alloc());
11701 : #else
11702 0 : auto* jumpTable = new (alloc()) OutOfLineSwitch<SwitchTableType::OutOfLine>(alloc());
11703 : #endif
11704 :
11705 : {
11706 : #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
11707 : // Inhibit pools within the following sequence because we are indexing into
11708 : // a pc relative table. The region will have one instruction for ma_ldr, one
11709 : // for breakpoint, and each table case takes one word.
11710 : AutoForbidPools afp(&masm, 1 + 1 + array->numElements());
11711 : #endif
11712 0 : jumpTable->jumpToCodeEntries(masm, index, temp0);
11713 :
11714 : // Add table entries if the table is inlined.
11715 0 : for (size_t i = 0, e = array->numElements(); i < e; i++)
11716 0 : jumpTable->addTableEntry(masm);
11717 : }
11718 :
11719 : // Add inlined code for loading arguments from where they are allocated.
11720 0 : for (size_t i = 0, e = array->numElements(); i < e; i++) {
11721 0 : MDefinition* elem = array->getElement(i);
11722 0 : ConstantOrRegister input;
11723 :
11724 0 : jumpTable->addCodeEntry(masm);
11725 0 : Register typeReg = Register::Invalid();
11726 0 : const LAllocation* a = lir->getOperand(1 + BOX_PIECES * i);
11727 0 : if (a->isBogus()) {
11728 0 : if (elem->type() == MIRType::Null) {
11729 0 : input = NullValue();
11730 0 : } else if (elem->type() == MIRType::Undefined) {
11731 0 : input = UndefinedValue();
11732 0 : } else if (elem->isConstant() && elem->isEmittedAtUses()) {
11733 0 : input = elem->toConstant()->toJSValue();
11734 : } else {
11735 0 : MOZ_CRASH("Unsupported element constant allocation.");
11736 : }
11737 0 : } else if (a->isMemory()) {
11738 0 : if (elem->type() == MIRType::Double) {
11739 0 : masm.loadDouble(ToAddress(a), tempD);
11740 0 : input = TypedOrValueRegister(elem->type(), AnyRegister(tempD));
11741 0 : } else if (elem->type() == MIRType::Value) {
11742 0 : typeReg = temp0;
11743 0 : masm.loadPtr(ToAddress(a), temp0);
11744 : #ifdef JS_PUNBOX64
11745 0 : input = TypedOrValueRegister(ValueOperand(temp0));
11746 : #endif
11747 : } else {
11748 0 : typeReg = temp0;
11749 0 : size_t width = StackSlotAllocator::width(LDefinition::TypeFrom(elem->type()));
11750 0 : if (width == 4)
11751 0 : masm.load32(ToAddress(a), temp0);
11752 0 : else if (width == 8)
11753 0 : masm.loadPtr(ToAddress(a), temp0);
11754 : else
11755 0 : MOZ_CRASH("Unsupported load size");
11756 0 : input = TypedOrValueRegister(elem->type(), AnyRegister(typeReg));
11757 : }
11758 0 : } else if (a->isGeneralReg()) {
11759 0 : typeReg = ToRegister(a);
11760 0 : input = TypedOrValueRegister(elem->type(), AnyRegister(typeReg));
11761 : #ifdef JS_PUNBOX64
11762 0 : if (elem->type() != MIRType::Value)
11763 0 : input = TypedOrValueRegister(elem->type(), AnyRegister(typeReg));
11764 : else
11765 0 : input = TypedOrValueRegister(ValueOperand(typeReg));
11766 : #else
11767 : if (elem->type() != MIRType::Value)
11768 : input = TypedOrValueRegister(elem->type(), AnyRegister(typeReg));
11769 : #endif
11770 0 : } else if (a->isFloatReg()) {
11771 0 : input = TypedOrValueRegister(elem->type(), AnyRegister(ToFloatRegister(a)));
11772 0 : } else if (a->isConstantValue()) {
11773 0 : input = a->toConstant()->toJSValue();
11774 : } else {
11775 0 : MOZ_CRASH("Unsupported element allocation.");
11776 : }
11777 :
11778 : #ifdef JS_NUNBOX32
11779 : if (elem->type() == MIRType::Value) {
11780 : static_assert(TYPE_INDEX == 0, "Unexpected type allocation index");
11781 : static_assert(PAYLOAD_INDEX == 1, "Unexpected payload allocation index");
11782 : const LAllocation* a1 = lir->getOperand(1 + BOX_PIECES * i + 1);
11783 : MOZ_ASSERT(!a1->isBogus());
11784 : MOZ_ASSERT(typeReg != Register::Invalid());
11785 : if (a1->isMemory()) {
11786 : masm.loadPtr(ToAddress(a1), temp1);
11787 : input = TypedOrValueRegister(ValueOperand(typeReg, temp1));
11788 : } else if (a1->isGeneralReg()) {
11789 : input = TypedOrValueRegister(ValueOperand(typeReg, ToRegister(a1)));
11790 : } else {
11791 : MOZ_CRASH("Unsupported Value allocation.");
11792 : }
11793 : } else {
11794 : MOZ_ASSERT(lir->getOperand(1 + BOX_PIECES * i + 1)->isBogus());
11795 : }
11796 : #endif
11797 0 : masm.moveValue(input, out);
11798 :
11799 : // For the last entry, fall-through.
11800 0 : if (i + 1 < e)
11801 0 : masm.jump(&join);
11802 : }
11803 :
11804 0 : addOutOfLineCode(jumpTable, lir->mir());
11805 0 : masm.bind(&join);
11806 0 : }
11807 :
11808 : template <typename T>
11809 : static inline void
11810 0 : StoreToTypedArray(MacroAssembler& masm, Scalar::Type writeType, const LAllocation* value,
11811 : const T& dest, unsigned numElems = 0)
11812 : {
11813 0 : if (Scalar::isSimdType(writeType) ||
11814 0 : writeType == Scalar::Float32 ||
11815 : writeType == Scalar::Float64)
11816 : {
11817 0 : masm.storeToTypedFloatArray(writeType, ToFloatRegister(value), dest, numElems);
11818 : } else {
11819 0 : if (value->isConstant())
11820 0 : masm.storeToTypedIntArray(writeType, Imm32(ToInt32(value)), dest);
11821 : else
11822 0 : masm.storeToTypedIntArray(writeType, ToRegister(value), dest);
11823 : }
11824 0 : }
11825 :
11826 : void
11827 0 : CodeGenerator::visitStoreUnboxedScalar(LStoreUnboxedScalar* lir)
11828 : {
11829 0 : Register elements = ToRegister(lir->elements());
11830 0 : const LAllocation* value = lir->value();
11831 :
11832 0 : const MStoreUnboxedScalar* mir = lir->mir();
11833 :
11834 0 : Scalar::Type writeType = mir->writeType();
11835 0 : unsigned numElems = mir->numElems();
11836 :
11837 0 : int width = Scalar::byteSize(mir->storageType());
11838 :
11839 0 : if (lir->index()->isConstant()) {
11840 0 : Address dest(elements, ToInt32(lir->index()) * width + mir->offsetAdjustment());
11841 0 : StoreToTypedArray(masm, writeType, value, dest, numElems);
11842 : } else {
11843 : BaseIndex dest(elements, ToRegister(lir->index()), ScaleFromElemWidth(width),
11844 0 : mir->offsetAdjustment());
11845 0 : StoreToTypedArray(masm, writeType, value, dest, numElems);
11846 : }
11847 0 : }
11848 :
11849 : void
11850 0 : CodeGenerator::visitStoreTypedArrayElementHole(LStoreTypedArrayElementHole* lir)
11851 : {
11852 0 : Register elements = ToRegister(lir->elements());
11853 0 : const LAllocation* value = lir->value();
11854 :
11855 0 : Scalar::Type arrayType = lir->mir()->arrayType();
11856 0 : int width = Scalar::byteSize(arrayType);
11857 :
11858 0 : Register index = ToRegister(lir->index());
11859 0 : const LAllocation* length = lir->length();
11860 0 : Register spectreTemp = ToTempRegisterOrInvalid(lir->spectreTemp());
11861 :
11862 0 : Label skip;
11863 0 : if (length->isRegister())
11864 0 : masm.spectreBoundsCheck32(index, ToRegister(length), spectreTemp, &skip);
11865 : else
11866 0 : masm.spectreBoundsCheck32(index, ToAddress(length), spectreTemp, &skip);
11867 :
11868 0 : BaseIndex dest(elements, index, ScaleFromElemWidth(width));
11869 0 : StoreToTypedArray(masm, arrayType, value, dest);
11870 :
11871 0 : masm.bind(&skip);
11872 0 : }
11873 :
11874 : void
11875 0 : CodeGenerator::visitAtomicIsLockFree(LAtomicIsLockFree* lir)
11876 : {
11877 0 : Register value = ToRegister(lir->value());
11878 0 : Register output = ToRegister(lir->output());
11879 :
11880 : // Keep this in sync with isLockfreeJS() in jit/AtomicOperations.h.
11881 0 : MOZ_ASSERT(AtomicOperations::isLockfreeJS(1)); // Implementation artifact
11882 0 : MOZ_ASSERT(AtomicOperations::isLockfreeJS(2)); // Implementation artifact
11883 0 : MOZ_ASSERT(AtomicOperations::isLockfreeJS(4)); // Spec requirement
11884 0 : MOZ_ASSERT(!AtomicOperations::isLockfreeJS(8)); // Implementation invariant, for now
11885 :
11886 0 : Label Ldone, Lfailed;
11887 0 : masm.move32(Imm32(1), output);
11888 0 : masm.branch32(Assembler::Equal, value, Imm32(4), &Ldone);
11889 0 : masm.branch32(Assembler::Equal, value, Imm32(2), &Ldone);
11890 0 : masm.branch32(Assembler::Equal, value, Imm32(1), &Ldone);
11891 0 : masm.move32(Imm32(0), output);
11892 0 : masm.bind(&Ldone);
11893 0 : }
11894 :
11895 : void
11896 0 : CodeGenerator::visitGuardSharedTypedArray(LGuardSharedTypedArray* guard)
11897 : {
11898 0 : Register obj = ToRegister(guard->input());
11899 0 : Register tmp = ToRegister(guard->tempInt());
11900 :
11901 : // The shared-memory flag is a bit in the ObjectElements header
11902 : // that is set if the TypedArray is mapping a SharedArrayBuffer.
11903 : // The flag is set at construction and does not change subsequently.
11904 0 : masm.loadPtr(Address(obj, TypedArrayObject::offsetOfElements()), tmp);
11905 0 : masm.load32(Address(tmp, ObjectElements::offsetOfFlags()), tmp);
11906 0 : bailoutTest32(Assembler::Zero, tmp, Imm32(ObjectElements::SHARED_MEMORY), guard->snapshot());
11907 0 : }
11908 :
11909 : void
11910 0 : CodeGenerator::visitClampIToUint8(LClampIToUint8* lir)
11911 : {
11912 0 : Register output = ToRegister(lir->output());
11913 0 : MOZ_ASSERT(output == ToRegister(lir->input()));
11914 0 : masm.clampIntToUint8(output);
11915 0 : }
11916 :
11917 : void
11918 0 : CodeGenerator::visitClampDToUint8(LClampDToUint8* lir)
11919 : {
11920 0 : FloatRegister input = ToFloatRegister(lir->input());
11921 0 : Register output = ToRegister(lir->output());
11922 0 : masm.clampDoubleToUint8(input, output);
11923 0 : }
11924 :
11925 : void
11926 0 : CodeGenerator::visitClampVToUint8(LClampVToUint8* lir)
11927 : {
11928 0 : ValueOperand operand = ToValue(lir, LClampVToUint8::Input);
11929 0 : FloatRegister tempFloat = ToFloatRegister(lir->tempFloat());
11930 0 : Register output = ToRegister(lir->output());
11931 0 : MDefinition* input = lir->mir()->input();
11932 :
11933 : Label* stringEntry;
11934 : Label* stringRejoin;
11935 0 : if (input->mightBeType(MIRType::String)) {
11936 0 : OutOfLineCode* oolString = oolCallVM(StringToNumberInfo, lir, ArgList(output),
11937 0 : StoreFloatRegisterTo(tempFloat));
11938 0 : stringEntry = oolString->entry();
11939 0 : stringRejoin = oolString->rejoin();
11940 : } else {
11941 : stringEntry = nullptr;
11942 : stringRejoin = nullptr;
11943 : }
11944 :
11945 0 : Label fails;
11946 0 : masm.clampValueToUint8(operand, input,
11947 : stringEntry, stringRejoin,
11948 0 : output, tempFloat, output, &fails);
11949 :
11950 0 : bailoutFrom(&fails, lir->snapshot());
11951 0 : }
11952 :
11953 : void
11954 20 : CodeGenerator::visitInCache(LInCache* ins)
11955 : {
11956 0 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
11957 :
11958 60 : ConstantOrRegister key = toConstantOrRegister(ins, LInCache::LHS, ins->mir()->key()->type());
11959 0 : Register object = ToRegister(ins->rhs());
11960 40 : Register output = ToRegister(ins->output());
11961 0 : Register temp = ToRegister(ins->temp());
11962 :
11963 0 : IonInIC cache(liveRegs, key, object, output, temp);
11964 0 : addIC(ins, allocateIC(cache));
11965 0 : }
11966 :
11967 : typedef bool (*OperatorInIFn)(JSContext*, uint32_t, HandleObject, bool*);
11968 0 : static const VMFunction OperatorInIInfo = FunctionInfo<OperatorInIFn>(OperatorInI, "OperatorInI");
11969 :
11970 : void
11971 0 : CodeGenerator::visitInArray(LInArray* lir)
11972 : {
11973 0 : const MInArray* mir = lir->mir();
11974 0 : Register elements = ToRegister(lir->elements());
11975 0 : Register initLength = ToRegister(lir->initLength());
11976 0 : Register output = ToRegister(lir->output());
11977 :
11978 : // When the array is not packed we need to do a hole check in addition to the bounds check.
11979 0 : Label falseBranch, done, trueBranch;
11980 :
11981 0 : OutOfLineCode* ool = nullptr;
11982 0 : Label* failedInitLength = &falseBranch;
11983 :
11984 0 : if (lir->index()->isConstant()) {
11985 0 : int32_t index = ToInt32(lir->index());
11986 :
11987 0 : MOZ_ASSERT_IF(index < 0, mir->needsNegativeIntCheck());
11988 0 : if (mir->needsNegativeIntCheck()) {
11989 0 : ool = oolCallVM(OperatorInIInfo, lir,
11990 0 : ArgList(Imm32(index), ToRegister(lir->object())),
11991 0 : StoreRegisterTo(output));
11992 0 : failedInitLength = ool->entry();
11993 : }
11994 :
11995 0 : masm.branch32(Assembler::BelowOrEqual, initLength, Imm32(index), failedInitLength);
11996 0 : if (mir->needsHoleCheck()) {
11997 : NativeObject::elementsSizeMustNotOverflow();
11998 0 : Address address = Address(elements, index * sizeof(Value));
11999 0 : masm.branchTestMagic(Assembler::Equal, address, &falseBranch);
12000 : }
12001 : } else {
12002 0 : Label negativeIntCheck;
12003 0 : Register index = ToRegister(lir->index());
12004 :
12005 0 : if (mir->needsNegativeIntCheck())
12006 0 : failedInitLength = &negativeIntCheck;
12007 :
12008 0 : masm.branch32(Assembler::BelowOrEqual, initLength, index, failedInitLength);
12009 0 : if (mir->needsHoleCheck()) {
12010 0 : BaseIndex address = BaseIndex(elements, ToRegister(lir->index()), TimesEight);
12011 0 : masm.branchTestMagic(Assembler::Equal, address, &falseBranch);
12012 : }
12013 0 : masm.jump(&trueBranch);
12014 :
12015 0 : if (mir->needsNegativeIntCheck()) {
12016 0 : masm.bind(&negativeIntCheck);
12017 0 : ool = oolCallVM(OperatorInIInfo, lir,
12018 0 : ArgList(index, ToRegister(lir->object())),
12019 0 : StoreRegisterTo(output));
12020 :
12021 0 : masm.branch32(Assembler::LessThan, index, Imm32(0), ool->entry());
12022 0 : masm.jump(&falseBranch);
12023 : }
12024 : }
12025 :
12026 0 : masm.bind(&trueBranch);
12027 0 : masm.move32(Imm32(1), output);
12028 0 : masm.jump(&done);
12029 :
12030 0 : masm.bind(&falseBranch);
12031 0 : masm.move32(Imm32(0), output);
12032 0 : masm.bind(&done);
12033 :
12034 0 : if (ool)
12035 0 : masm.bind(ool->rejoin());
12036 0 : }
12037 :
12038 : void
12039 0 : CodeGenerator::visitInstanceOfO(LInstanceOfO* ins)
12040 : {
12041 0 : emitInstanceOf(ins, ins->mir()->prototypeObject());
12042 0 : }
12043 :
12044 : void
12045 0 : CodeGenerator::visitInstanceOfV(LInstanceOfV* ins)
12046 : {
12047 0 : emitInstanceOf(ins, ins->mir()->prototypeObject());
12048 0 : }
12049 :
12050 : // Wrap IsDelegateOfObject, which takes a JSObject*, not a HandleObject
12051 : static bool
12052 0 : IsDelegateObject(JSContext* cx, HandleObject protoObj, HandleObject obj, bool* res)
12053 : {
12054 0 : return IsDelegateOfObject(cx, protoObj, obj, res);
12055 : }
12056 :
12057 : typedef bool (*IsDelegateObjectFn)(JSContext*, HandleObject, HandleObject, bool*);
12058 1 : static const VMFunction IsDelegateObjectInfo =
12059 0 : FunctionInfo<IsDelegateObjectFn>(IsDelegateObject, "IsDelegateObject");
12060 :
12061 : void
12062 0 : CodeGenerator::emitInstanceOf(LInstruction* ins, JSObject* prototypeObject)
12063 : {
12064 : // This path implements fun_hasInstance when the function's prototype is
12065 : // known to be prototypeObject.
12066 :
12067 0 : Label done;
12068 0 : Register output = ToRegister(ins->getDef(0));
12069 :
12070 : // If the lhs is a primitive, the result is false.
12071 : Register objReg;
12072 0 : if (ins->isInstanceOfV()) {
12073 0 : Label isObject;
12074 0 : ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LHS);
12075 0 : masm.branchTestObject(Assembler::Equal, lhsValue, &isObject);
12076 0 : masm.mov(ImmWord(0), output);
12077 0 : masm.jump(&done);
12078 0 : masm.bind(&isObject);
12079 0 : objReg = masm.extractObject(lhsValue, output);
12080 : } else {
12081 0 : objReg = ToRegister(ins->toInstanceOfO()->lhs());
12082 : }
12083 :
12084 : // Crawl the lhs's prototype chain in a loop to search for prototypeObject.
12085 : // This follows the main loop of js::IsDelegate, though additionally breaks
12086 : // out of the loop on Proxy::LazyProto.
12087 :
12088 : // Load the lhs's prototype.
12089 0 : masm.loadObjProto(objReg, output);
12090 :
12091 0 : Label testLazy;
12092 : {
12093 0 : Label loopPrototypeChain;
12094 0 : masm.bind(&loopPrototypeChain);
12095 :
12096 : // Test for the target prototype object.
12097 0 : Label notPrototypeObject;
12098 0 : masm.branchPtr(Assembler::NotEqual, output, ImmGCPtr(prototypeObject), ¬PrototypeObject);
12099 0 : masm.mov(ImmWord(1), output);
12100 0 : masm.jump(&done);
12101 0 : masm.bind(¬PrototypeObject);
12102 :
12103 0 : MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
12104 :
12105 : // Test for nullptr or Proxy::LazyProto
12106 0 : masm.branchPtr(Assembler::BelowOrEqual, output, ImmWord(1), &testLazy);
12107 :
12108 : // Load the current object's prototype.
12109 0 : masm.loadObjProto(output, output);
12110 :
12111 0 : masm.jump(&loopPrototypeChain);
12112 : }
12113 :
12114 : // Make a VM call if an object with a lazy proto was found on the prototype
12115 : // chain. This currently occurs only for cross compartment wrappers, which
12116 : // we do not expect to be compared with non-wrapper functions from this
12117 : // compartment. Otherwise, we stopped on a nullptr prototype and the output
12118 : // register is already correct.
12119 :
12120 0 : OutOfLineCode* ool = oolCallVM(IsDelegateObjectInfo, ins,
12121 0 : ArgList(ImmGCPtr(prototypeObject), objReg),
12122 0 : StoreRegisterTo(output));
12123 :
12124 : // Regenerate the original lhs object for the VM call.
12125 0 : Label regenerate, *lazyEntry;
12126 0 : if (objReg != output) {
12127 0 : lazyEntry = ool->entry();
12128 : } else {
12129 0 : masm.bind(®enerate);
12130 0 : lazyEntry = ®enerate;
12131 0 : if (ins->isInstanceOfV()) {
12132 0 : ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LHS);
12133 0 : objReg = masm.extractObject(lhsValue, output);
12134 : } else {
12135 0 : objReg = ToRegister(ins->toInstanceOfO()->lhs());
12136 : }
12137 0 : MOZ_ASSERT(objReg == output);
12138 0 : masm.jump(ool->entry());
12139 : }
12140 :
12141 0 : masm.bind(&testLazy);
12142 0 : masm.branchPtr(Assembler::Equal, output, ImmWord(1), lazyEntry);
12143 :
12144 0 : masm.bind(&done);
12145 0 : masm.bind(ool->rejoin());
12146 0 : }
12147 :
12148 : void
12149 0 : CodeGenerator::visitInstanceOfCache(LInstanceOfCache* ins)
12150 : {
12151 : // The Lowering ensures that RHS is an object, and that LHS is a value.
12152 0 : LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
12153 0 : TypedOrValueRegister lhs = TypedOrValueRegister(ToValue(ins, LInstanceOfCache::LHS));
12154 0 : Register rhs = ToRegister(ins->rhs());
12155 0 : Register output = ToRegister(ins->output());
12156 :
12157 0 : IonInstanceOfIC ic(liveRegs, lhs, rhs, output);
12158 0 : addIC(ins, allocateIC(ic));
12159 0 : }
12160 :
12161 : void
12162 0 : CodeGenerator::visitGetDOMProperty(LGetDOMProperty* ins)
12163 : {
12164 0 : const Register JSContextReg = ToRegister(ins->getJSContextReg());
12165 0 : const Register ObjectReg = ToRegister(ins->getObjectReg());
12166 0 : const Register PrivateReg = ToRegister(ins->getPrivReg());
12167 0 : const Register ValueReg = ToRegister(ins->getValueReg());
12168 :
12169 0 : Label haveValue;
12170 0 : if (ins->mir()->valueMayBeInSlot()) {
12171 0 : size_t slot = ins->mir()->domMemberSlotIndex();
12172 : // It's a bit annoying to redo these slot calculations, which duplcate
12173 : // LSlots and a few other things like that, but I'm not sure there's a
12174 : // way to reuse those here.
12175 : //
12176 : // If this ever gets fixed to work with proxies (by not assuming that
12177 : // reserved slot indices, which is what domMemberSlotIndex() returns,
12178 : // match fixed slot indices), we can reenable MGetDOMProperty for
12179 : // proxies in IonBuilder.
12180 0 : if (slot < NativeObject::MAX_FIXED_SLOTS) {
12181 0 : masm.loadValue(Address(ObjectReg, NativeObject::getFixedSlotOffset(slot)),
12182 0 : JSReturnOperand);
12183 : } else {
12184 : // It's a dynamic slot.
12185 0 : slot -= NativeObject::MAX_FIXED_SLOTS;
12186 : // Use PrivateReg as a scratch register for the slots pointer.
12187 0 : masm.loadPtr(Address(ObjectReg, NativeObject::offsetOfSlots()),
12188 0 : PrivateReg);
12189 0 : masm.loadValue(Address(PrivateReg, slot*sizeof(js::Value)),
12190 0 : JSReturnOperand);
12191 : }
12192 0 : masm.branchTestUndefined(Assembler::NotEqual, JSReturnOperand, &haveValue);
12193 : }
12194 :
12195 0 : DebugOnly<uint32_t> initialStack = masm.framePushed();
12196 :
12197 0 : masm.checkStackAlignment();
12198 :
12199 : // Make space for the outparam. Pre-initialize it to UndefinedValue so we
12200 : // can trace it at GC time.
12201 0 : masm.Push(UndefinedValue());
12202 : // We pass the pointer to our out param as an instance of
12203 : // JSJitGetterCallArgs, since on the binary level it's the same thing.
12204 : JS_STATIC_ASSERT(sizeof(JSJitGetterCallArgs) == sizeof(Value*));
12205 0 : masm.moveStackPtrTo(ValueReg);
12206 :
12207 0 : masm.Push(ObjectReg);
12208 :
12209 0 : LoadDOMPrivate(masm, ObjectReg, PrivateReg, ins->mir()->objectKind());
12210 :
12211 : // Rooting will happen at GC time.
12212 0 : masm.moveStackPtrTo(ObjectReg);
12213 :
12214 0 : uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg);
12215 0 : masm.loadJSContext(JSContextReg);
12216 0 : masm.enterFakeExitFrame(JSContextReg, JSContextReg, ExitFrameType::IonDOMGetter);
12217 :
12218 0 : markSafepointAt(safepointOffset, ins);
12219 :
12220 0 : masm.setupUnalignedABICall(JSContextReg);
12221 0 : masm.loadJSContext(JSContextReg);
12222 0 : masm.passABIArg(JSContextReg);
12223 0 : masm.passABIArg(ObjectReg);
12224 0 : masm.passABIArg(PrivateReg);
12225 0 : masm.passABIArg(ValueReg);
12226 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ins->mir()->fun()), MoveOp::GENERAL,
12227 0 : CheckUnsafeCallWithABI::DontCheckHasExitFrame);
12228 :
12229 0 : if (ins->mir()->isInfallible()) {
12230 0 : masm.loadValue(Address(masm.getStackPointer(), IonDOMExitFrameLayout::offsetOfResult()),
12231 0 : JSReturnOperand);
12232 : } else {
12233 0 : masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
12234 :
12235 0 : masm.loadValue(Address(masm.getStackPointer(), IonDOMExitFrameLayout::offsetOfResult()),
12236 0 : JSReturnOperand);
12237 : }
12238 :
12239 : // Until C++ code is instrumented against Spectre, prevent speculative
12240 : // execution from returning any private data.
12241 0 : if (JitOptions.spectreJitToCxxCalls && ins->mir()->hasLiveDefUses())
12242 0 : masm.speculationBarrier();
12243 :
12244 0 : masm.adjustStack(IonDOMExitFrameLayout::Size());
12245 :
12246 0 : masm.bind(&haveValue);
12247 :
12248 0 : MOZ_ASSERT(masm.framePushed() == initialStack);
12249 0 : }
12250 :
12251 : void
12252 0 : CodeGenerator::visitGetDOMMemberV(LGetDOMMemberV* ins)
12253 : {
12254 : // It's simpler to duplicate visitLoadFixedSlotV here than it is to try to
12255 : // use an LLoadFixedSlotV or some subclass of it for this case: that would
12256 : // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then
12257 : // we'd have to duplicate a bunch of stuff we now get for free from
12258 : // MGetDOMProperty.
12259 : //
12260 : // If this ever gets fixed to work with proxies (by not assuming that
12261 : // reserved slot indices, which is what domMemberSlotIndex() returns,
12262 : // match fixed slot indices), we can reenable MGetDOMMember for
12263 : // proxies in IonBuilder.
12264 0 : Register object = ToRegister(ins->object());
12265 0 : size_t slot = ins->mir()->domMemberSlotIndex();
12266 0 : ValueOperand result = ToOutValue(ins);
12267 :
12268 0 : masm.loadValue(Address(object, NativeObject::getFixedSlotOffset(slot)), result);
12269 0 : }
12270 :
12271 : void
12272 0 : CodeGenerator::visitGetDOMMemberT(LGetDOMMemberT* ins)
12273 : {
12274 : // It's simpler to duplicate visitLoadFixedSlotT here than it is to try to
12275 : // use an LLoadFixedSlotT or some subclass of it for this case: that would
12276 : // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then
12277 : // we'd have to duplicate a bunch of stuff we now get for free from
12278 : // MGetDOMProperty.
12279 : //
12280 : // If this ever gets fixed to work with proxies (by not assuming that
12281 : // reserved slot indices, which is what domMemberSlotIndex() returns,
12282 : // match fixed slot indices), we can reenable MGetDOMMember for
12283 : // proxies in IonBuilder.
12284 0 : Register object = ToRegister(ins->object());
12285 0 : size_t slot = ins->mir()->domMemberSlotIndex();
12286 0 : AnyRegister result = ToAnyRegister(ins->getDef(0));
12287 0 : MIRType type = ins->mir()->type();
12288 :
12289 0 : masm.loadUnboxedValue(Address(object, NativeObject::getFixedSlotOffset(slot)), type, result);
12290 0 : }
12291 :
12292 : void
12293 0 : CodeGenerator::visitSetDOMProperty(LSetDOMProperty* ins)
12294 : {
12295 0 : const Register JSContextReg = ToRegister(ins->getJSContextReg());
12296 0 : const Register ObjectReg = ToRegister(ins->getObjectReg());
12297 0 : const Register PrivateReg = ToRegister(ins->getPrivReg());
12298 0 : const Register ValueReg = ToRegister(ins->getValueReg());
12299 :
12300 0 : DebugOnly<uint32_t> initialStack = masm.framePushed();
12301 :
12302 0 : masm.checkStackAlignment();
12303 :
12304 : // Push the argument. Rooting will happen at GC time.
12305 0 : ValueOperand argVal = ToValue(ins, LSetDOMProperty::Value);
12306 0 : masm.Push(argVal);
12307 : // We pass the pointer to our out param as an instance of
12308 : // JSJitGetterCallArgs, since on the binary level it's the same thing.
12309 : JS_STATIC_ASSERT(sizeof(JSJitSetterCallArgs) == sizeof(Value*));
12310 0 : masm.moveStackPtrTo(ValueReg);
12311 :
12312 0 : masm.Push(ObjectReg);
12313 :
12314 0 : LoadDOMPrivate(masm, ObjectReg, PrivateReg, ins->mir()->objectKind());
12315 :
12316 : // Rooting will happen at GC time.
12317 0 : masm.moveStackPtrTo(ObjectReg);
12318 :
12319 0 : uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg);
12320 0 : masm.loadJSContext(JSContextReg);
12321 0 : masm.enterFakeExitFrame(JSContextReg, JSContextReg, ExitFrameType::IonDOMSetter);
12322 :
12323 0 : markSafepointAt(safepointOffset, ins);
12324 :
12325 0 : masm.setupUnalignedABICall(JSContextReg);
12326 0 : masm.loadJSContext(JSContextReg);
12327 0 : masm.passABIArg(JSContextReg);
12328 0 : masm.passABIArg(ObjectReg);
12329 0 : masm.passABIArg(PrivateReg);
12330 0 : masm.passABIArg(ValueReg);
12331 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ins->mir()->fun()), MoveOp::GENERAL,
12332 0 : CheckUnsafeCallWithABI::DontCheckHasExitFrame);
12333 :
12334 0 : masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
12335 :
12336 0 : masm.adjustStack(IonDOMExitFrameLayout::Size());
12337 :
12338 0 : MOZ_ASSERT(masm.framePushed() == initialStack);
12339 0 : }
12340 :
12341 : class OutOfLineIsCallable : public OutOfLineCodeBase<CodeGenerator>
12342 : {
12343 : Register object_;
12344 : Register output_;
12345 :
12346 : public:
12347 : OutOfLineIsCallable(Register object, Register output)
12348 14 : : object_(object), output_(output)
12349 : { }
12350 :
12351 7 : void accept(CodeGenerator* codegen) override {
12352 7 : codegen->visitOutOfLineIsCallable(this);
12353 0 : }
12354 : Register object() const {
12355 : return object_;
12356 : }
12357 : Register output() const {
12358 : return output_;
12359 : }
12360 : };
12361 :
12362 : template <CodeGenerator::CallableOrConstructor mode>
12363 : void
12364 12 : CodeGenerator::emitIsCallableOrConstructor(Register object, Register output, Label* failure)
12365 : {
12366 48 : Label notFunction, hasCOps, done;
12367 12 : masm.loadObjClassUnsafe(object, output);
12368 :
12369 : // Just skim proxies off. Their notion of isCallable()/isConstructor() is
12370 : // more complicated.
12371 0 : masm.branchTestClassIsProxy(true, output, failure);
12372 :
12373 : // An object is callable iff:
12374 : // is<JSFunction>() || (getClass()->cOps && getClass()->cOps->call).
12375 : // An object is constructor iff:
12376 : // ((is<JSFunction>() && as<JSFunction>().isConstructor) ||
12377 : // (getClass()->cOps && getClass()->cOps->construct)).
12378 24 : masm.branchPtr(Assembler::NotEqual, output, ImmPtr(&JSFunction::class_), ¬Function);
12379 : if (mode == Callable) {
12380 14 : masm.move32(Imm32(1), output);
12381 : } else {
12382 10 : Label notConstructor;
12383 0 : masm.load16ZeroExtend(Address(object, JSFunction::offsetOfFlags()), output);
12384 5 : masm.and32(Imm32(JSFunction::CONSTRUCTOR), output);
12385 0 : masm.branchTest32(Assembler::Zero, output, output, ¬Constructor);
12386 10 : masm.move32(Imm32(1), output);
12387 0 : masm.jump(&done);
12388 0 : masm.bind(¬Constructor);
12389 0 : masm.move32(Imm32(0), output);
12390 : }
12391 0 : masm.jump(&done);
12392 :
12393 0 : masm.bind(¬Function);
12394 0 : masm.branchPtr(Assembler::NonZero, Address(output, offsetof(js::Class, cOps)),
12395 : ImmPtr(nullptr), &hasCOps);
12396 0 : masm.move32(Imm32(0), output);
12397 24 : masm.jump(&done);
12398 :
12399 0 : masm.bind(&hasCOps);
12400 12 : masm.loadPtr(Address(output, offsetof(js::Class, cOps)), output);
12401 : size_t opsOffset = mode == Callable
12402 : ? offsetof(js::ClassOps, call)
12403 12 : : offsetof(js::ClassOps, construct);
12404 0 : masm.cmpPtrSet(Assembler::NonZero, Address(output, opsOffset),
12405 : ImmPtr(nullptr), output);
12406 :
12407 12 : masm.bind(&done);
12408 0 : }
12409 :
12410 : void
12411 5 : CodeGenerator::visitIsCallableO(LIsCallableO* ins)
12412 : {
12413 0 : Register object = ToRegister(ins->object());
12414 10 : Register output = ToRegister(ins->output());
12415 :
12416 0 : OutOfLineIsCallable* ool = new(alloc()) OutOfLineIsCallable(object, output);
12417 10 : addOutOfLineCode(ool, ins->mir());
12418 :
12419 0 : emitIsCallableOrConstructor<Callable>(object, output, ool->entry());
12420 :
12421 0 : masm.bind(ool->rejoin());
12422 0 : }
12423 :
12424 : void
12425 2 : CodeGenerator::visitIsCallableV(LIsCallableV* ins)
12426 : {
12427 0 : ValueOperand val = ToValue(ins, LIsCallableV::Value);
12428 4 : Register output = ToRegister(ins->output());
12429 4 : Register temp = ToRegister(ins->temp());
12430 :
12431 4 : Label notObject;
12432 0 : masm.branchTestObject(Assembler::NotEqual, val, ¬Object);
12433 0 : masm.unboxObject(val, temp);
12434 :
12435 8 : OutOfLineIsCallable* ool = new(alloc()) OutOfLineIsCallable(temp, output);
12436 0 : addOutOfLineCode(ool, ins->mir());
12437 :
12438 0 : emitIsCallableOrConstructor<Callable>(temp, output, ool->entry());
12439 6 : masm.jump(ool->rejoin());
12440 :
12441 0 : masm.bind(¬Object);
12442 4 : masm.move32(Imm32(0), output);
12443 :
12444 0 : masm.bind(ool->rejoin());
12445 2 : }
12446 :
12447 : void
12448 7 : CodeGenerator::visitOutOfLineIsCallable(OutOfLineIsCallable* ool)
12449 : {
12450 0 : Register object = ool->object();
12451 7 : Register output = ool->output();
12452 :
12453 0 : saveVolatile(output);
12454 7 : masm.setupUnalignedABICall(output);
12455 0 : masm.passABIArg(object);
12456 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ObjectIsCallable));
12457 7 : masm.storeCallBoolResult(output);
12458 0 : restoreVolatile(output);
12459 0 : masm.jump(ool->rejoin());
12460 0 : }
12461 :
12462 : typedef bool (*CheckIsCallableFn)(JSContext*, HandleValue, CheckIsCallableKind);
12463 0 : static const VMFunction CheckIsCallableInfo =
12464 0 : FunctionInfo<CheckIsCallableFn>(CheckIsCallable, "CheckIsCallable");
12465 :
12466 : void
12467 0 : CodeGenerator::visitCheckIsCallable(LCheckIsCallable* ins)
12468 : {
12469 0 : ValueOperand checkValue = ToValue(ins, LCheckIsCallable::CheckValue);
12470 0 : Register temp = ToRegister(ins->temp());
12471 :
12472 : // OOL code is used in the following 2 cases:
12473 : // * checkValue is not callable
12474 : // * checkValue is proxy and it's unknown whether it's callable or not
12475 : // CheckIsCallable checks if passed value is callable, regardless of the
12476 : // cases above. IsCallable operation is not observable and checking it
12477 : // again doesn't matter.
12478 0 : OutOfLineCode* ool = oolCallVM(CheckIsCallableInfo, ins,
12479 0 : ArgList(checkValue, Imm32(ins->mir()->checkKind())),
12480 0 : StoreNothing());
12481 :
12482 0 : masm.branchTestObject(Assembler::NotEqual, checkValue, ool->entry());
12483 :
12484 0 : Register object = masm.extractObject(checkValue, temp);
12485 0 : emitIsCallableOrConstructor<Callable>(object, temp, ool->entry());
12486 :
12487 0 : masm.branchTest32(Assembler::Zero, temp, temp, ool->entry());
12488 :
12489 0 : masm.bind(ool->rejoin());
12490 0 : }
12491 :
12492 : class OutOfLineIsConstructor : public OutOfLineCodeBase<CodeGenerator>
12493 : {
12494 : LIsConstructor* ins_;
12495 :
12496 : public:
12497 : explicit OutOfLineIsConstructor(LIsConstructor* ins)
12498 10 : : ins_(ins)
12499 : { }
12500 :
12501 5 : void accept(CodeGenerator* codegen) override {
12502 5 : codegen->visitOutOfLineIsConstructor(this);
12503 0 : }
12504 : LIsConstructor* ins() const {
12505 : return ins_;
12506 : }
12507 : };
12508 :
12509 : void
12510 5 : CodeGenerator::visitIsConstructor(LIsConstructor* ins)
12511 : {
12512 10 : Register object = ToRegister(ins->object());
12513 10 : Register output = ToRegister(ins->output());
12514 :
12515 0 : OutOfLineIsConstructor* ool = new(alloc()) OutOfLineIsConstructor(ins);
12516 10 : addOutOfLineCode(ool, ins->mir());
12517 :
12518 0 : emitIsCallableOrConstructor<Constructor>(object, output, ool->entry());
12519 :
12520 0 : masm.bind(ool->rejoin());
12521 0 : }
12522 :
12523 : void
12524 5 : CodeGenerator::visitOutOfLineIsConstructor(OutOfLineIsConstructor* ool)
12525 : {
12526 0 : LIsConstructor* ins = ool->ins();
12527 10 : Register object = ToRegister(ins->object());
12528 10 : Register output = ToRegister(ins->output());
12529 :
12530 10 : saveVolatile(output);
12531 0 : masm.setupUnalignedABICall(output);
12532 0 : masm.passABIArg(object);
12533 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, ObjectIsConstructor));
12534 5 : masm.storeCallBoolResult(output);
12535 0 : restoreVolatile(output);
12536 0 : masm.jump(ool->rejoin());
12537 0 : }
12538 :
12539 : typedef bool (*IsArrayFn)(JSContext*, HandleObject, bool*);
12540 0 : static const VMFunction IsArrayInfo = FunctionInfo<IsArrayFn>(JS::IsArray, "IsArray");
12541 :
12542 : static void
12543 7 : EmitObjectIsArray(MacroAssembler& masm, OutOfLineCode* ool, Register obj, Register output,
12544 : Label* notArray = nullptr)
12545 : {
12546 7 : masm.loadObjClassUnsafe(obj, output);
12547 :
12548 0 : Label isArray;
12549 14 : masm.branchPtr(Assembler::Equal, output, ImmPtr(&ArrayObject::class_), &isArray);
12550 :
12551 : // Branch to OOL path if it's a proxy.
12552 14 : masm.branchTestClassIsProxy(true, output, ool->entry());
12553 :
12554 0 : if (notArray)
12555 0 : masm.bind(notArray);
12556 14 : masm.move32(Imm32(0), output);
12557 0 : masm.jump(ool->rejoin());
12558 :
12559 0 : masm.bind(&isArray);
12560 0 : masm.move32(Imm32(1), output);
12561 :
12562 0 : masm.bind(ool->rejoin());
12563 7 : }
12564 :
12565 : void
12566 7 : CodeGenerator::visitIsArrayO(LIsArrayO* lir)
12567 : {
12568 0 : Register object = ToRegister(lir->object());
12569 14 : Register output = ToRegister(lir->output());
12570 :
12571 0 : OutOfLineCode* ool = oolCallVM(IsArrayInfo, lir, ArgList(object),
12572 21 : StoreRegisterTo(output));
12573 0 : EmitObjectIsArray(masm, ool, object, output);
12574 0 : }
12575 :
12576 : void
12577 0 : CodeGenerator::visitIsArrayV(LIsArrayV* lir)
12578 : {
12579 0 : ValueOperand val = ToValue(lir, LIsArrayV::Value);
12580 0 : Register output = ToRegister(lir->output());
12581 0 : Register temp = ToRegister(lir->temp());
12582 :
12583 0 : Label notArray;
12584 0 : masm.branchTestObject(Assembler::NotEqual, val, ¬Array);
12585 0 : masm.unboxObject(val, temp);
12586 :
12587 0 : OutOfLineCode* ool = oolCallVM(IsArrayInfo, lir, ArgList(temp),
12588 0 : StoreRegisterTo(output));
12589 0 : EmitObjectIsArray(masm, ool, temp, output, ¬Array);
12590 0 : }
12591 :
12592 : void
12593 0 : CodeGenerator::visitIsTypedArray(LIsTypedArray* lir)
12594 : {
12595 0 : Register object = ToRegister(lir->object());
12596 0 : Register output = ToRegister(lir->output());
12597 :
12598 0 : Label notTypedArray;
12599 0 : Label done;
12600 :
12601 : static_assert(Scalar::Int8 == 0, "Int8 is the first typed array class");
12602 : static_assert((Scalar::Uint8Clamped - Scalar::Int8) == Scalar::MaxTypedArrayViewType - 1,
12603 : "Uint8Clamped is the last typed array class");
12604 0 : const Class* firstTypedArrayClass = TypedArrayObject::classForType(Scalar::Int8);
12605 0 : const Class* lastTypedArrayClass = TypedArrayObject::classForType(Scalar::Uint8Clamped);
12606 :
12607 0 : masm.loadObjClassUnsafe(object, output);
12608 0 : masm.branchPtr(Assembler::Below, output, ImmPtr(firstTypedArrayClass), ¬TypedArray);
12609 0 : masm.branchPtr(Assembler::Above, output, ImmPtr(lastTypedArrayClass), ¬TypedArray);
12610 :
12611 0 : masm.move32(Imm32(1), output);
12612 0 : masm.jump(&done);
12613 0 : masm.bind(¬TypedArray);
12614 0 : masm.move32(Imm32(0), output);
12615 0 : masm.bind(&done);
12616 0 : }
12617 :
12618 : void
12619 0 : CodeGenerator::visitIsObject(LIsObject* ins)
12620 : {
12621 0 : Register output = ToRegister(ins->output());
12622 0 : ValueOperand value = ToValue(ins, LIsObject::Input);
12623 0 : masm.testObjectSet(Assembler::Equal, value, output);
12624 0 : }
12625 :
12626 : void
12627 0 : CodeGenerator::visitIsObjectAndBranch(LIsObjectAndBranch* ins)
12628 : {
12629 0 : ValueOperand value = ToValue(ins, LIsObjectAndBranch::Input);
12630 4 : testObjectEmitBranch(Assembler::Equal, value, ins->ifTrue(), ins->ifFalse());
12631 2 : }
12632 :
12633 : void
12634 0 : CodeGenerator::loadOutermostJSScript(Register reg)
12635 : {
12636 : // The "outermost" JSScript means the script that we are compiling
12637 : // basically; this is not always the script associated with the
12638 : // current basic block, which might be an inlined script.
12639 :
12640 0 : MIRGraph& graph = current->mir()->graph();
12641 0 : MBasicBlock* entryBlock = graph.entryBlock();
12642 0 : masm.movePtr(ImmGCPtr(entryBlock->info().script()), reg);
12643 0 : }
12644 :
12645 : void
12646 0 : CodeGenerator::loadJSScriptForBlock(MBasicBlock* block, Register reg)
12647 : {
12648 : // The current JSScript means the script for the current
12649 : // basic block. This may be an inlined script.
12650 :
12651 0 : JSScript* script = block->info().script();
12652 0 : masm.movePtr(ImmGCPtr(script), reg);
12653 0 : }
12654 :
12655 : void
12656 0 : CodeGenerator::visitHasClass(LHasClass* ins)
12657 : {
12658 0 : Register lhs = ToRegister(ins->lhs());
12659 0 : Register output = ToRegister(ins->output());
12660 :
12661 0 : masm.loadObjClassUnsafe(lhs, output);
12662 0 : masm.cmpPtrSet(Assembler::Equal, output, ImmPtr(ins->mir()->getClass()), output);
12663 0 : }
12664 :
12665 : void
12666 0 : CodeGenerator::visitGuardToClass(LGuardToClass* ins)
12667 : {
12668 0 : Register lhs = ToRegister(ins->lhs());
12669 16 : Register output = ToRegister(ins->output());
12670 16 : Register temp = ToRegister(ins->temp());
12671 :
12672 16 : Label notEqual;
12673 :
12674 0 : masm.branchTestObjClass(Assembler::NotEqual, lhs, ins->mir()->getClass(), temp,
12675 0 : output, ¬Equal);
12676 16 : masm.mov(lhs, output);
12677 :
12678 16 : if (ins->mir()->type() == MIRType::Object) {
12679 : // Can't return null-return here, so bail
12680 0 : bailoutFrom(¬Equal, ins->snapshot());
12681 : } else {
12682 0 : Label done;
12683 0 : masm.jump(&done);
12684 :
12685 0 : masm.bind(¬Equal);
12686 0 : masm.mov(ImmPtr(0), output);
12687 :
12688 0 : masm.bind(&done);
12689 : }
12690 1 : }
12691 :
12692 : typedef JSString* (*ObjectClassToStringFn)(JSContext*, HandleObject);
12693 1 : static const VMFunction ObjectClassToStringInfo =
12694 3 : FunctionInfo<ObjectClassToStringFn>(js::ObjectClassToString, "ObjectClassToString");
12695 :
12696 : void
12697 0 : CodeGenerator::visitObjectClassToString(LObjectClassToString* lir)
12698 : {
12699 0 : pushArg(ToRegister(lir->object()));
12700 0 : callVM(ObjectClassToStringInfo, lir);
12701 0 : }
12702 :
12703 : void
12704 0 : CodeGenerator::visitWasmParameter(LWasmParameter* lir)
12705 : {
12706 0 : }
12707 :
12708 : void
12709 0 : CodeGenerator::visitWasmParameterI64(LWasmParameterI64* lir)
12710 : {
12711 0 : }
12712 :
12713 : void
12714 0 : CodeGenerator::visitWasmReturn(LWasmReturn* lir)
12715 : {
12716 : // Don't emit a jump to the return label if this is the last block.
12717 0 : if (current->mir() != *gen->graph().poBegin())
12718 0 : masm.jump(&returnLabel_);
12719 0 : }
12720 :
12721 : void
12722 0 : CodeGenerator::visitWasmReturnI64(LWasmReturnI64* lir)
12723 : {
12724 : // Don't emit a jump to the return label if this is the last block.
12725 0 : if (current->mir() != *gen->graph().poBegin())
12726 0 : masm.jump(&returnLabel_);
12727 0 : }
12728 :
12729 : void
12730 0 : CodeGenerator::visitWasmReturnVoid(LWasmReturnVoid* lir)
12731 : {
12732 : // Don't emit a jump to the return label if this is the last block.
12733 0 : if (current->mir() != *gen->graph().poBegin())
12734 0 : masm.jump(&returnLabel_);
12735 0 : }
12736 :
12737 : void
12738 0 : CodeGenerator::emitAssertRangeI(const Range* r, Register input)
12739 : {
12740 : // Check the lower bound.
12741 0 : if (r->hasInt32LowerBound() && r->lower() > INT32_MIN) {
12742 0 : Label success;
12743 0 : masm.branch32(Assembler::GreaterThanOrEqual, input, Imm32(r->lower()), &success);
12744 0 : masm.assumeUnreachable("Integer input should be equal or higher than Lowerbound.");
12745 0 : masm.bind(&success);
12746 : }
12747 :
12748 : // Check the upper bound.
12749 0 : if (r->hasInt32UpperBound() && r->upper() < INT32_MAX) {
12750 0 : Label success;
12751 0 : masm.branch32(Assembler::LessThanOrEqual, input, Imm32(r->upper()), &success);
12752 0 : masm.assumeUnreachable("Integer input should be lower or equal than Upperbound.");
12753 0 : masm.bind(&success);
12754 : }
12755 :
12756 : // For r->canHaveFractionalPart(), r->canBeNegativeZero(), and
12757 : // r->exponent(), there's nothing to check, because if we ended up in the
12758 : // integer range checking code, the value is already in an integer register
12759 : // in the integer range.
12760 0 : }
12761 :
12762 : void
12763 0 : CodeGenerator::emitAssertRangeD(const Range* r, FloatRegister input, FloatRegister temp)
12764 : {
12765 : // Check the lower bound.
12766 0 : if (r->hasInt32LowerBound()) {
12767 0 : Label success;
12768 0 : masm.loadConstantDouble(r->lower(), temp);
12769 0 : if (r->canBeNaN())
12770 0 : masm.branchDouble(Assembler::DoubleUnordered, input, input, &success);
12771 0 : masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp, &success);
12772 0 : masm.assumeUnreachable("Double input should be equal or higher than Lowerbound.");
12773 0 : masm.bind(&success);
12774 : }
12775 : // Check the upper bound.
12776 0 : if (r->hasInt32UpperBound()) {
12777 0 : Label success;
12778 0 : masm.loadConstantDouble(r->upper(), temp);
12779 0 : if (r->canBeNaN())
12780 0 : masm.branchDouble(Assembler::DoubleUnordered, input, input, &success);
12781 0 : masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp, &success);
12782 0 : masm.assumeUnreachable("Double input should be lower or equal than Upperbound.");
12783 0 : masm.bind(&success);
12784 : }
12785 :
12786 : // This code does not yet check r->canHaveFractionalPart(). This would require new
12787 : // assembler interfaces to make rounding instructions available.
12788 :
12789 0 : if (!r->canBeNegativeZero()) {
12790 0 : Label success;
12791 :
12792 : // First, test for being equal to 0.0, which also includes -0.0.
12793 0 : masm.loadConstantDouble(0.0, temp);
12794 0 : masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, input, temp, &success);
12795 :
12796 : // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0 is
12797 : // -Infinity instead of Infinity.
12798 0 : masm.loadConstantDouble(1.0, temp);
12799 0 : masm.divDouble(input, temp);
12800 0 : masm.branchDouble(Assembler::DoubleGreaterThan, temp, input, &success);
12801 :
12802 0 : masm.assumeUnreachable("Input shouldn't be negative zero.");
12803 :
12804 0 : masm.bind(&success);
12805 : }
12806 :
12807 0 : if (!r->hasInt32Bounds() && !r->canBeInfiniteOrNaN() &&
12808 0 : r->exponent() < FloatingPoint<double>::kExponentBias)
12809 : {
12810 : // Check the bounds implied by the maximum exponent.
12811 0 : Label exponentLoOk;
12812 0 : masm.loadConstantDouble(pow(2.0, r->exponent() + 1), temp);
12813 0 : masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentLoOk);
12814 0 : masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp, &exponentLoOk);
12815 0 : masm.assumeUnreachable("Check for exponent failed.");
12816 0 : masm.bind(&exponentLoOk);
12817 :
12818 0 : Label exponentHiOk;
12819 0 : masm.loadConstantDouble(-pow(2.0, r->exponent() + 1), temp);
12820 0 : masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentHiOk);
12821 0 : masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp, &exponentHiOk);
12822 0 : masm.assumeUnreachable("Check for exponent failed.");
12823 0 : masm.bind(&exponentHiOk);
12824 0 : } else if (!r->hasInt32Bounds() && !r->canBeNaN()) {
12825 : // If we think the value can't be NaN, check that it isn't.
12826 0 : Label notnan;
12827 0 : masm.branchDouble(Assembler::DoubleOrdered, input, input, ¬nan);
12828 0 : masm.assumeUnreachable("Input shouldn't be NaN.");
12829 0 : masm.bind(¬nan);
12830 :
12831 : // If we think the value also can't be an infinity, check that it isn't.
12832 0 : if (!r->canBeInfiniteOrNaN()) {
12833 0 : Label notposinf;
12834 0 : masm.loadConstantDouble(PositiveInfinity<double>(), temp);
12835 0 : masm.branchDouble(Assembler::DoubleLessThan, input, temp, ¬posinf);
12836 0 : masm.assumeUnreachable("Input shouldn't be +Inf.");
12837 0 : masm.bind(¬posinf);
12838 :
12839 0 : Label notneginf;
12840 0 : masm.loadConstantDouble(NegativeInfinity<double>(), temp);
12841 0 : masm.branchDouble(Assembler::DoubleGreaterThan, input, temp, ¬neginf);
12842 0 : masm.assumeUnreachable("Input shouldn't be -Inf.");
12843 0 : masm.bind(¬neginf);
12844 : }
12845 : }
12846 0 : }
12847 :
12848 : void
12849 0 : CodeGenerator::visitAssertResultV(LAssertResultV* ins)
12850 : {
12851 : #ifdef DEBUG
12852 0 : const ValueOperand value = ToValue(ins, LAssertResultV::Input);
12853 0 : emitAssertResultV(value, ins->mirRaw()->resultTypeSet());
12854 : #else
12855 : MOZ_CRASH("LAssertResultV is debug only");
12856 : #endif
12857 0 : }
12858 :
12859 : void
12860 0 : CodeGenerator::visitAssertResultT(LAssertResultT* ins)
12861 : {
12862 : #ifdef DEBUG
12863 0 : Register input = ToRegister(ins->input());
12864 0 : MDefinition* mir = ins->mirRaw();
12865 0 : emitAssertObjectOrStringResult(input, mir->type(), mir->resultTypeSet());
12866 : #else
12867 : MOZ_CRASH("LAssertResultT is debug only");
12868 : #endif
12869 0 : }
12870 :
12871 : void
12872 0 : CodeGenerator::visitAssertRangeI(LAssertRangeI* ins)
12873 : {
12874 0 : Register input = ToRegister(ins->input());
12875 0 : const Range* r = ins->range();
12876 :
12877 0 : emitAssertRangeI(r, input);
12878 0 : }
12879 :
12880 : void
12881 0 : CodeGenerator::visitAssertRangeD(LAssertRangeD* ins)
12882 : {
12883 0 : FloatRegister input = ToFloatRegister(ins->input());
12884 0 : FloatRegister temp = ToFloatRegister(ins->temp());
12885 0 : const Range* r = ins->range();
12886 :
12887 0 : emitAssertRangeD(r, input, temp);
12888 0 : }
12889 :
12890 : void
12891 0 : CodeGenerator::visitAssertRangeF(LAssertRangeF* ins)
12892 : {
12893 0 : FloatRegister input = ToFloatRegister(ins->input());
12894 0 : FloatRegister temp = ToFloatRegister(ins->temp());
12895 0 : FloatRegister temp2 = ToFloatRegister(ins->temp2());
12896 :
12897 0 : const Range* r = ins->range();
12898 :
12899 0 : masm.convertFloat32ToDouble(input, temp);
12900 0 : emitAssertRangeD(r, temp, temp2);
12901 0 : }
12902 :
12903 : void
12904 0 : CodeGenerator::visitAssertRangeV(LAssertRangeV* ins)
12905 : {
12906 0 : const Range* r = ins->range();
12907 0 : const ValueOperand value = ToValue(ins, LAssertRangeV::Input);
12908 0 : Label done;
12909 :
12910 : {
12911 0 : ScratchTagScope tag(masm, value);
12912 0 : masm.splitTagForTest(value, tag);
12913 :
12914 : {
12915 0 : Label isNotInt32;
12916 0 : masm.branchTestInt32(Assembler::NotEqual, tag, &isNotInt32);
12917 : {
12918 0 : ScratchTagScopeRelease _(&tag);
12919 0 : Register unboxInt32 = ToTempUnboxRegister(ins->temp());
12920 0 : Register input = masm.extractInt32(value, unboxInt32);
12921 0 : emitAssertRangeI(r, input);
12922 0 : masm.jump(&done);
12923 : }
12924 0 : masm.bind(&isNotInt32);
12925 : }
12926 :
12927 : {
12928 0 : Label isNotDouble;
12929 0 : masm.branchTestDouble(Assembler::NotEqual, tag, &isNotDouble);
12930 : {
12931 0 : ScratchTagScopeRelease _(&tag);
12932 0 : FloatRegister input = ToFloatRegister(ins->floatTemp1());
12933 0 : FloatRegister temp = ToFloatRegister(ins->floatTemp2());
12934 0 : masm.unboxDouble(value, input);
12935 0 : emitAssertRangeD(r, input, temp);
12936 0 : masm.jump(&done);
12937 : }
12938 0 : masm.bind(&isNotDouble);
12939 : }
12940 : }
12941 :
12942 0 : masm.assumeUnreachable("Incorrect range for Value.");
12943 0 : masm.bind(&done);
12944 0 : }
12945 :
12946 : void
12947 0 : CodeGenerator::visitInterruptCheck(LInterruptCheck* lir)
12948 : {
12949 0 : OutOfLineCode* ool = oolCallVM(InterruptCheckInfo, lir, ArgList(), StoreNothing());
12950 :
12951 218 : const void* interruptAddr = gen->runtime->addressOfInterruptBits();
12952 0 : masm.branch32(Assembler::NotEqual, AbsoluteAddress(interruptAddr), Imm32(0), ool->entry());
12953 109 : masm.bind(ool->rejoin());
12954 0 : }
12955 :
12956 : void
12957 0 : CodeGenerator::visitWasmInterruptCheck(LWasmInterruptCheck* lir)
12958 : {
12959 0 : MOZ_ASSERT(gen->compilingWasm());
12960 :
12961 0 : masm.wasmInterruptCheck(ToRegister(lir->tlsPtr()), lir->mir()->bytecodeOffset());
12962 0 : }
12963 :
12964 : void
12965 0 : CodeGenerator::visitWasmTrap(LWasmTrap* lir)
12966 : {
12967 0 : MOZ_ASSERT(gen->compilingWasm());
12968 0 : const MWasmTrap* mir = lir->mir();
12969 :
12970 0 : masm.wasmTrap(mir->trap(), mir->bytecodeOffset());
12971 0 : }
12972 :
12973 : void
12974 0 : CodeGenerator::visitWasmBoundsCheck(LWasmBoundsCheck* ins)
12975 : {
12976 : #ifdef WASM_HUGE_MEMORY
12977 0 : MOZ_CRASH("No wasm bounds check for huge memory");
12978 : #else
12979 : const MWasmBoundsCheck* mir = ins->mir();
12980 : Register ptr = ToRegister(ins->ptr());
12981 : Register boundsCheckLimit = ToRegister(ins->boundsCheckLimit());
12982 : Label ok;
12983 : masm.wasmBoundsCheck(Assembler::Below, ptr, boundsCheckLimit, &ok);
12984 : masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
12985 : masm.bind(&ok);
12986 : #endif
12987 : }
12988 :
12989 : void
12990 0 : CodeGenerator::visitWasmAlignmentCheck(LWasmAlignmentCheck* ins)
12991 : {
12992 0 : const MWasmAlignmentCheck* mir = ins->mir();
12993 0 : Register ptr = ToRegister(ins->ptr());
12994 0 : Label ok;
12995 0 : masm.branchTest32(Assembler::Zero, ptr, Imm32(mir->byteSize() - 1), &ok);
12996 0 : masm.wasmTrap(wasm::Trap::UnalignedAccess, mir->bytecodeOffset());
12997 0 : masm.bind(&ok);
12998 0 : }
12999 :
13000 : void
13001 0 : CodeGenerator::visitWasmLoadTls(LWasmLoadTls* ins)
13002 : {
13003 0 : switch (ins->mir()->type()) {
13004 : case MIRType::Pointer:
13005 0 : masm.loadPtr(Address(ToRegister(ins->tlsPtr()), ins->mir()->offset()),
13006 0 : ToRegister(ins->output()));
13007 0 : break;
13008 : case MIRType::Int32:
13009 0 : masm.load32(Address(ToRegister(ins->tlsPtr()), ins->mir()->offset()),
13010 0 : ToRegister(ins->output()));
13011 0 : break;
13012 : default:
13013 0 : MOZ_CRASH("MIRType not supported in WasmLoadTls");
13014 : }
13015 0 : }
13016 :
13017 : typedef bool (*RecompileFn)(JSContext*);
13018 1 : static const VMFunction RecompileFnInfo = FunctionInfo<RecompileFn>(Recompile, "Recompile");
13019 :
13020 : typedef bool (*ForcedRecompileFn)(JSContext*);
13021 1 : static const VMFunction ForcedRecompileFnInfo =
13022 3 : FunctionInfo<ForcedRecompileFn>(ForcedRecompile, "ForcedRecompile");
13023 :
13024 : void
13025 8 : CodeGenerator::visitRecompileCheck(LRecompileCheck* ins)
13026 : {
13027 0 : Label done;
13028 16 : Register tmp = ToRegister(ins->scratch());
13029 : OutOfLineCode* ool;
13030 0 : if (ins->mir()->forceRecompilation())
13031 8 : ool = oolCallVM(ForcedRecompileFnInfo, ins, ArgList(), StoreRegisterTo(tmp));
13032 : else
13033 0 : ool = oolCallVM(RecompileFnInfo, ins, ArgList(), StoreRegisterTo(tmp));
13034 :
13035 : // Check if warm-up counter is high enough.
13036 0 : AbsoluteAddress warmUpCount = AbsoluteAddress(ins->mir()->script()->addressOfWarmUpCounter());
13037 16 : if (ins->mir()->increaseWarmUpCounter()) {
13038 0 : masm.load32(warmUpCount, tmp);
13039 0 : masm.add32(Imm32(1), tmp);
13040 0 : masm.store32(tmp, warmUpCount);
13041 0 : masm.branch32(Assembler::BelowOrEqual, tmp, Imm32(ins->mir()->recompileThreshold()), &done);
13042 : } else {
13043 1 : masm.branch32(Assembler::BelowOrEqual, warmUpCount, Imm32(ins->mir()->recompileThreshold()),
13044 1 : &done);
13045 : }
13046 :
13047 : // Check if not yet recompiling.
13048 0 : CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), tmp);
13049 0 : masm.propagateOOM(ionScriptLabels_.append(label));
13050 32 : masm.branch32(Assembler::Equal,
13051 24 : Address(tmp, IonScript::offsetOfRecompiling()),
13052 : Imm32(0),
13053 0 : ool->entry());
13054 0 : masm.bind(ool->rejoin());
13055 0 : masm.bind(&done);
13056 0 : }
13057 :
13058 : void
13059 0 : CodeGenerator::visitLexicalCheck(LLexicalCheck* ins)
13060 : {
13061 0 : ValueOperand inputValue = ToValue(ins, LLexicalCheck::Input);
13062 4 : Label bail;
13063 2 : masm.branchTestMagicValue(Assembler::Equal, inputValue, JS_UNINITIALIZED_LEXICAL, &bail);
13064 0 : bailoutFrom(&bail, ins->snapshot());
13065 2 : }
13066 :
13067 : typedef bool (*ThrowRuntimeLexicalErrorFn)(JSContext*, unsigned);
13068 0 : static const VMFunction ThrowRuntimeLexicalErrorInfo =
13069 0 : FunctionInfo<ThrowRuntimeLexicalErrorFn>(ThrowRuntimeLexicalError, "ThrowRuntimeLexicalError");
13070 :
13071 : void
13072 0 : CodeGenerator::visitThrowRuntimeLexicalError(LThrowRuntimeLexicalError* ins)
13073 : {
13074 0 : pushArg(Imm32(ins->mir()->errorNumber()));
13075 0 : callVM(ThrowRuntimeLexicalErrorInfo, ins);
13076 0 : }
13077 :
13078 : typedef bool (*GlobalNameConflictsCheckFromIonFn)(JSContext*, HandleScript);
13079 0 : static const VMFunction GlobalNameConflictsCheckFromIonInfo =
13080 0 : FunctionInfo<GlobalNameConflictsCheckFromIonFn>(GlobalNameConflictsCheckFromIon,
13081 0 : "GlobalNameConflictsCheckFromIon");
13082 :
13083 : void
13084 0 : CodeGenerator::visitGlobalNameConflictsCheck(LGlobalNameConflictsCheck* ins)
13085 : {
13086 0 : pushArg(ImmGCPtr(ins->mirRaw()->block()->info().script()));
13087 0 : callVM(GlobalNameConflictsCheckFromIonInfo, ins);
13088 0 : }
13089 :
13090 : void
13091 0 : CodeGenerator::visitDebugger(LDebugger* ins)
13092 : {
13093 0 : Register cx = ToRegister(ins->getTemp(0));
13094 0 : Register temp = ToRegister(ins->getTemp(1));
13095 :
13096 0 : masm.loadJSContext(cx);
13097 0 : masm.setupUnalignedABICall(temp);
13098 0 : masm.passABIArg(cx);
13099 0 : masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, GlobalHasLiveOnDebuggerStatement));
13100 :
13101 0 : Label bail;
13102 0 : masm.branchIfTrueBool(ReturnReg, &bail);
13103 0 : bailoutFrom(&bail, ins->snapshot());
13104 0 : }
13105 :
13106 : void
13107 0 : CodeGenerator::visitNewTarget(LNewTarget *ins)
13108 : {
13109 0 : ValueOperand output = ToOutValue(ins);
13110 :
13111 : // if (isConstructing) output = argv[Max(numActualArgs, numFormalArgs)]
13112 0 : Label notConstructing, done;
13113 0 : Address calleeToken(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfCalleeToken());
13114 0 : masm.branchTestPtr(Assembler::Zero, calleeToken,
13115 0 : Imm32(CalleeToken_FunctionConstructing), ¬Constructing);
13116 :
13117 0 : Register argvLen = output.scratchReg();
13118 :
13119 0 : Address actualArgsPtr(masm.getStackPointer(), frameSize() + JitFrameLayout::offsetOfNumActualArgs());
13120 0 : masm.loadPtr(actualArgsPtr, argvLen);
13121 :
13122 0 : Label useNFormals;
13123 :
13124 0 : size_t numFormalArgs = ins->mirRaw()->block()->info().nargs();
13125 0 : masm.branchPtr(Assembler::Below, argvLen, Imm32(numFormalArgs),
13126 0 : &useNFormals);
13127 :
13128 0 : size_t argsOffset = frameSize() + JitFrameLayout::offsetOfActualArgs();
13129 : {
13130 0 : BaseValueIndex newTarget(masm.getStackPointer(), argvLen, argsOffset);
13131 0 : masm.loadValue(newTarget, output);
13132 0 : masm.jump(&done);
13133 : }
13134 :
13135 0 : masm.bind(&useNFormals);
13136 :
13137 : {
13138 0 : Address newTarget(masm.getStackPointer(), argsOffset + (numFormalArgs * sizeof(Value)));
13139 0 : masm.loadValue(newTarget, output);
13140 0 : masm.jump(&done);
13141 : }
13142 :
13143 : // else output = undefined
13144 0 : masm.bind(¬Constructing);
13145 0 : masm.moveValue(UndefinedValue(), output);
13146 0 : masm.bind(&done);
13147 0 : }
13148 :
13149 : void
13150 0 : CodeGenerator::visitCheckReturn(LCheckReturn* ins)
13151 : {
13152 0 : ValueOperand returnValue = ToValue(ins, LCheckReturn::ReturnValue);
13153 0 : ValueOperand thisValue = ToValue(ins, LCheckReturn::ThisValue);
13154 0 : Label bail, noChecks;
13155 0 : masm.branchTestObject(Assembler::Equal, returnValue, &noChecks);
13156 0 : masm.branchTestUndefined(Assembler::NotEqual, returnValue, &bail);
13157 0 : masm.branchTestMagicValue(Assembler::Equal, thisValue, JS_UNINITIALIZED_LEXICAL, &bail);
13158 0 : bailoutFrom(&bail, ins->snapshot());
13159 0 : masm.bind(&noChecks);
13160 0 : }
13161 :
13162 : typedef bool (*ThrowCheckIsObjectFn)(JSContext*, CheckIsObjectKind);
13163 1 : static const VMFunction ThrowCheckIsObjectInfo =
13164 1 : FunctionInfo<ThrowCheckIsObjectFn>(ThrowCheckIsObject, "ThrowCheckIsObject");
13165 :
13166 : void
13167 0 : CodeGenerator::visitCheckIsObj(LCheckIsObj* ins)
13168 : {
13169 0 : ValueOperand checkValue = ToValue(ins, LCheckIsObj::CheckValue);
13170 :
13171 0 : OutOfLineCode* ool = oolCallVM(ThrowCheckIsObjectInfo, ins,
13172 0 : ArgList(Imm32(ins->mir()->checkKind())),
13173 0 : StoreNothing());
13174 0 : masm.branchTestObject(Assembler::NotEqual, checkValue, ool->entry());
13175 0 : masm.bind(ool->rejoin());
13176 0 : }
13177 :
13178 : typedef bool (*ThrowObjCoercibleFn)(JSContext*, HandleValue);
13179 0 : static const VMFunction ThrowObjectCoercibleInfo =
13180 0 : FunctionInfo<ThrowObjCoercibleFn>(ThrowObjectCoercible, "ThrowObjectCoercible");
13181 :
13182 : void
13183 0 : CodeGenerator::visitCheckObjCoercible(LCheckObjCoercible* ins)
13184 : {
13185 0 : ValueOperand checkValue = ToValue(ins, LCheckObjCoercible::CheckValue);
13186 0 : Label fail, done;
13187 0 : masm.branchTestNull(Assembler::Equal, checkValue, &fail);
13188 0 : masm.branchTestUndefined(Assembler::NotEqual, checkValue, &done);
13189 0 : masm.bind(&fail);
13190 0 : pushArg(checkValue);
13191 0 : callVM(ThrowObjectCoercibleInfo, ins);
13192 0 : masm.bind(&done);
13193 0 : }
13194 :
13195 : typedef bool (*CheckSelfHostedFn)(JSContext*, HandleValue);
13196 0 : static const VMFunction CheckSelfHostedInfo =
13197 0 : FunctionInfo<CheckSelfHostedFn>(js::Debug_CheckSelfHosted, "Debug_CheckSelfHosted");
13198 :
13199 : void
13200 8 : CodeGenerator::visitDebugCheckSelfHosted(LDebugCheckSelfHosted* ins)
13201 : {
13202 0 : ValueOperand checkValue = ToValue(ins, LDebugCheckSelfHosted::CheckValue);
13203 16 : pushArg(checkValue);
13204 8 : callVM(CheckSelfHostedInfo, ins);
13205 0 : }
13206 :
13207 : void
13208 0 : CodeGenerator::visitRandom(LRandom* ins)
13209 : {
13210 : using mozilla::non_crypto::XorShift128PlusRNG;
13211 :
13212 0 : FloatRegister output = ToFloatRegister(ins->output());
13213 0 : Register tempReg = ToRegister(ins->temp0());
13214 :
13215 : #ifdef JS_PUNBOX64
13216 0 : Register64 s0Reg(ToRegister(ins->temp1()));
13217 0 : Register64 s1Reg(ToRegister(ins->temp2()));
13218 : #else
13219 : Register64 s0Reg(ToRegister(ins->temp1()), ToRegister(ins->temp2()));
13220 : Register64 s1Reg(ToRegister(ins->temp3()), ToRegister(ins->temp4()));
13221 : #endif
13222 :
13223 0 : const void* rng = gen->realm->addressOfRandomNumberGenerator();
13224 0 : masm.movePtr(ImmPtr(rng), tempReg);
13225 :
13226 : static_assert(sizeof(XorShift128PlusRNG) == 2 * sizeof(uint64_t),
13227 : "Code below assumes XorShift128PlusRNG contains two uint64_t values");
13228 :
13229 0 : Address state0Addr(tempReg, XorShift128PlusRNG::offsetOfState0());
13230 0 : Address state1Addr(tempReg, XorShift128PlusRNG::offsetOfState1());
13231 :
13232 : // uint64_t s1 = mState[0];
13233 0 : masm.load64(state0Addr, s1Reg);
13234 :
13235 : // s1 ^= s1 << 23;
13236 0 : masm.move64(s1Reg, s0Reg);
13237 0 : masm.lshift64(Imm32(23), s1Reg);
13238 0 : masm.xor64(s0Reg, s1Reg);
13239 :
13240 : // s1 ^= s1 >> 17
13241 0 : masm.move64(s1Reg, s0Reg);
13242 0 : masm.rshift64(Imm32(17), s1Reg);
13243 0 : masm.xor64(s0Reg, s1Reg);
13244 :
13245 : // const uint64_t s0 = mState[1];
13246 0 : masm.load64(state1Addr, s0Reg);
13247 :
13248 : // mState[0] = s0;
13249 0 : masm.store64(s0Reg, state0Addr);
13250 :
13251 : // s1 ^= s0
13252 0 : masm.xor64(s0Reg, s1Reg);
13253 :
13254 : // s1 ^= s0 >> 26
13255 0 : masm.rshift64(Imm32(26), s0Reg);
13256 0 : masm.xor64(s0Reg, s1Reg);
13257 :
13258 : // mState[1] = s1
13259 0 : masm.store64(s1Reg, state1Addr);
13260 :
13261 : // s1 += mState[0]
13262 0 : masm.load64(state0Addr, s0Reg);
13263 0 : masm.add64(s0Reg, s1Reg);
13264 :
13265 : // See comment in XorShift128PlusRNG::nextDouble().
13266 : static const int MantissaBits = FloatingPoint<double>::kExponentShift + 1;
13267 : static const double ScaleInv = double(1) / (1ULL << MantissaBits);
13268 :
13269 0 : masm.and64(Imm64((1ULL << MantissaBits) - 1), s1Reg);
13270 :
13271 0 : if (masm.convertUInt64ToDoubleNeedsTemp())
13272 0 : masm.convertUInt64ToDouble(s1Reg, output, tempReg);
13273 : else
13274 0 : masm.convertUInt64ToDouble(s1Reg, output, Register::Invalid());
13275 :
13276 : // output *= ScaleInv
13277 0 : masm.mulDoublePtr(ImmPtr(&ScaleInv), tempReg, output);
13278 0 : }
13279 :
13280 : void
13281 0 : CodeGenerator::visitSignExtendInt32(LSignExtendInt32* ins)
13282 : {
13283 0 : Register input = ToRegister(ins->input());
13284 0 : Register output = ToRegister(ins->output());
13285 :
13286 0 : switch (ins->mode()) {
13287 : case MSignExtendInt32::Byte:
13288 0 : masm.move8SignExtend(input, output);
13289 : break;
13290 : case MSignExtendInt32::Half:
13291 0 : masm.move16SignExtend(input, output);
13292 : break;
13293 : }
13294 0 : }
13295 :
13296 : void
13297 0 : CodeGenerator::visitRotate(LRotate* ins)
13298 : {
13299 0 : MRotate* mir = ins->mir();
13300 0 : Register input = ToRegister(ins->input());
13301 0 : Register dest = ToRegister(ins->output());
13302 :
13303 0 : const LAllocation* count = ins->count();
13304 0 : if (count->isConstant()) {
13305 0 : int32_t c = ToInt32(count) & 0x1F;
13306 0 : if (mir->isLeftRotate())
13307 0 : masm.rotateLeft(Imm32(c), input, dest);
13308 : else
13309 0 : masm.rotateRight(Imm32(c), input, dest);
13310 : } else {
13311 0 : Register creg = ToRegister(count);
13312 0 : if (mir->isLeftRotate())
13313 0 : masm.rotateLeft(creg, input, dest);
13314 : else
13315 0 : masm.rotateRight(creg, input, dest);
13316 : }
13317 0 : }
13318 :
13319 : class OutOfLineNaNToZero : public OutOfLineCodeBase<CodeGenerator>
13320 : {
13321 : LNaNToZero* lir_;
13322 :
13323 : public:
13324 : explicit OutOfLineNaNToZero(LNaNToZero* lir)
13325 0 : : lir_(lir)
13326 : {}
13327 :
13328 0 : void accept(CodeGenerator* codegen) override {
13329 0 : codegen->visitOutOfLineNaNToZero(this);
13330 0 : }
13331 : LNaNToZero* lir() const {
13332 : return lir_;
13333 : }
13334 : };
13335 :
13336 : void
13337 0 : CodeGenerator::visitOutOfLineNaNToZero(OutOfLineNaNToZero* ool)
13338 : {
13339 0 : FloatRegister output = ToFloatRegister(ool->lir()->output());
13340 0 : masm.loadConstantDouble(0.0, output);
13341 0 : masm.jump(ool->rejoin());
13342 0 : }
13343 :
13344 : void
13345 0 : CodeGenerator::visitNaNToZero(LNaNToZero* lir)
13346 : {
13347 0 : FloatRegister input = ToFloatRegister(lir->input());
13348 :
13349 0 : OutOfLineNaNToZero* ool = new(alloc()) OutOfLineNaNToZero(lir);
13350 0 : addOutOfLineCode(ool, lir->mir());
13351 :
13352 0 : if (lir->mir()->operandIsNeverNegativeZero()){
13353 0 : masm.branchDouble(Assembler::DoubleUnordered, input, input, ool->entry());
13354 : } else {
13355 0 : FloatRegister scratch = ToFloatRegister(lir->tempDouble());
13356 0 : masm.loadConstantDouble(0.0, scratch);
13357 0 : masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, scratch, ool->entry());
13358 : }
13359 0 : masm.bind(ool->rejoin());
13360 0 : }
13361 :
13362 : typedef bool (*FinishBoundFunctionInitFn)(JSContext* cx, HandleFunction bound,
13363 : HandleObject target, int32_t argCount);
13364 0 : static const VMFunction FinishBoundFunctionInitInfo =
13365 0 : FunctionInfo<FinishBoundFunctionInitFn>(JSFunction::finishBoundFunctionInit,
13366 1 : "JSFunction::finishBoundFunctionInit");
13367 :
13368 : void
13369 0 : CodeGenerator::visitFinishBoundFunctionInit(LFinishBoundFunctionInit* lir)
13370 : {
13371 0 : Register bound = ToRegister(lir->bound());
13372 0 : Register target = ToRegister(lir->target());
13373 0 : Register argCount = ToRegister(lir->argCount());
13374 0 : Register temp1 = ToRegister(lir->temp1());
13375 0 : Register temp2 = ToRegister(lir->temp2());
13376 :
13377 0 : OutOfLineCode* ool = oolCallVM(FinishBoundFunctionInitInfo, lir,
13378 0 : ArgList(bound, target, argCount), StoreNothing());
13379 0 : Label* slowPath = ool->entry();
13380 :
13381 0 : const size_t boundLengthOffset = FunctionExtended::offsetOfExtendedSlot(BOUND_FUN_LENGTH_SLOT);
13382 :
13383 : // Take the slow path if the target is not a JSFunction.
13384 0 : masm.branchTestObjClass(Assembler::NotEqual, target, &JSFunction::class_, temp1, target,
13385 0 : slowPath);
13386 :
13387 : // Take the slow path if we'd need to adjust the [[Prototype]].
13388 0 : masm.loadObjProto(bound, temp1);
13389 0 : masm.loadObjProto(target, temp2);
13390 0 : masm.branchPtr(Assembler::NotEqual, temp1, temp2, slowPath);
13391 :
13392 : // Get the function flags.
13393 0 : masm.load16ZeroExtend(Address(target, JSFunction::offsetOfFlags()), temp1);
13394 :
13395 : // Functions with lazy scripts don't store their length.
13396 : // If the length or name property is resolved, it might be shadowed.
13397 0 : masm.branchTest32(Assembler::NonZero,
13398 : temp1,
13399 : Imm32(JSFunction::INTERPRETED_LAZY |
13400 : JSFunction::RESOLVED_NAME |
13401 : JSFunction::RESOLVED_LENGTH),
13402 0 : slowPath);
13403 :
13404 0 : Label notBoundTarget, loadName;
13405 0 : masm.branchTest32(Assembler::Zero, temp1, Imm32(JSFunction::BOUND_FUN), ¬BoundTarget);
13406 : {
13407 : // Call into the VM if the target's name atom contains the bound
13408 : // function prefix.
13409 0 : masm.branchTest32(Assembler::NonZero, temp1,
13410 0 : Imm32(JSFunction::HAS_BOUND_FUNCTION_NAME_PREFIX), slowPath);
13411 :
13412 : // We also take the slow path when target's length isn't an int32.
13413 0 : masm.branchTestInt32(Assembler::NotEqual, Address(target, boundLengthOffset), slowPath);
13414 :
13415 : // Bound functions reuse HAS_GUESSED_ATOM for HAS_BOUND_FUNCTION_NAME_PREFIX,
13416 : // so skip the guessed atom check below.
13417 : static_assert(JSFunction::HAS_BOUND_FUNCTION_NAME_PREFIX == JSFunction::HAS_GUESSED_ATOM,
13418 : "HAS_BOUND_FUNCTION_NAME_PREFIX is shared with HAS_GUESSED_ATOM");
13419 0 : masm.jump(&loadName);
13420 : }
13421 0 : masm.bind(¬BoundTarget);
13422 :
13423 0 : Label guessed, hasName;
13424 0 : masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSFunction::HAS_GUESSED_ATOM), &guessed);
13425 0 : masm.bind(&loadName);
13426 0 : masm.loadPtr(Address(target, JSFunction::offsetOfAtom()), temp2);
13427 0 : masm.branchTestPtr(Assembler::NonZero, temp2, temp2, &hasName);
13428 : {
13429 0 : masm.bind(&guessed);
13430 :
13431 : // Unnamed class expression don't have a name property. To avoid
13432 : // looking it up from the prototype chain, we take the slow path here.
13433 0 : masm.branchFunctionKind(Assembler::Equal, JSFunction::ClassConstructor, target, temp2,
13434 0 : slowPath);
13435 :
13436 : // An absent name property defaults to the empty string.
13437 0 : const JSAtomState& names = gen->runtime->names();
13438 0 : masm.movePtr(ImmGCPtr(names.empty), temp2);
13439 : }
13440 0 : masm.bind(&hasName);
13441 :
13442 : // Store the target's name atom in the bound function as is.
13443 0 : masm.storePtr(temp2, Address(bound, JSFunction::offsetOfAtom()));
13444 :
13445 : // Set the BOUND_FN flag and, if the target is a constructor, the
13446 : // CONSTRUCTOR flag.
13447 0 : Label isConstructor, boundFlagsComputed;
13448 0 : masm.load16ZeroExtend(Address(bound, JSFunction::offsetOfFlags()), temp2);
13449 0 : masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSFunction::CONSTRUCTOR), &isConstructor);
13450 : {
13451 0 : masm.or32(Imm32(JSFunction::BOUND_FUN), temp2);
13452 0 : masm.jump(&boundFlagsComputed);
13453 : }
13454 0 : masm.bind(&isConstructor);
13455 : {
13456 0 : masm.or32(Imm32(JSFunction::BOUND_FUN | JSFunction::CONSTRUCTOR), temp2);
13457 : }
13458 0 : masm.bind(&boundFlagsComputed);
13459 0 : masm.store16(temp2, Address(bound, JSFunction::offsetOfFlags()));
13460 :
13461 : // Load the target function's length.
13462 0 : Label isInterpreted, isBound, lengthLoaded;
13463 0 : masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSFunction::BOUND_FUN), &isBound);
13464 0 : masm.branchTest32(Assembler::NonZero, temp1, Imm32(JSFunction::INTERPRETED), &isInterpreted);
13465 : {
13466 : // Load the length property of a native function.
13467 0 : masm.load16ZeroExtend(Address(target, JSFunction::offsetOfNargs()), temp1);
13468 0 : masm.jump(&lengthLoaded);
13469 : }
13470 0 : masm.bind(&isBound);
13471 : {
13472 : // Load the length property of a bound function.
13473 0 : masm.unboxInt32(Address(target, boundLengthOffset), temp1);
13474 0 : masm.jump(&lengthLoaded);
13475 : }
13476 0 : masm.bind(&isInterpreted);
13477 : {
13478 : // Load the length property of an interpreted function.
13479 0 : masm.loadPtr(Address(target, JSFunction::offsetOfScript()), temp1);
13480 0 : masm.load16ZeroExtend(Address(temp1, JSScript::offsetOfFunLength()), temp1);
13481 : }
13482 0 : masm.bind(&lengthLoaded);
13483 :
13484 : // Compute the bound function length: Max(0, target.length - argCount).
13485 0 : Label nonNegative;
13486 0 : masm.sub32(argCount, temp1);
13487 0 : masm.branch32(Assembler::GreaterThanOrEqual, temp1, Imm32(0), &nonNegative);
13488 0 : masm.move32(Imm32(0), temp1);
13489 0 : masm.bind(&nonNegative);
13490 :
13491 : // Store the bound function's length into the extended slot.
13492 0 : masm.storeValue(JSVAL_TYPE_INT32, temp1, Address(bound, boundLengthOffset));
13493 :
13494 0 : masm.bind(ool->rejoin());
13495 0 : }
13496 :
13497 : void
13498 0 : CodeGenerator::visitIsPackedArray(LIsPackedArray* lir)
13499 : {
13500 0 : Register array = ToRegister(lir->array());
13501 0 : Register output = ToRegister(lir->output());
13502 0 : Register elementsTemp = ToRegister(lir->temp());
13503 :
13504 0 : Label notPacked, done;
13505 :
13506 : // Load elements and length.
13507 0 : masm.loadPtr(Address(array, NativeObject::offsetOfElements()), elementsTemp);
13508 0 : masm.load32(Address(elementsTemp, ObjectElements::offsetOfLength()), output);
13509 :
13510 : // Test length == initializedLength.
13511 0 : Address initLength(elementsTemp, ObjectElements::offsetOfInitializedLength());
13512 0 : masm.branch32(Assembler::NotEqual, initLength, output, ¬Packed);
13513 :
13514 0 : masm.move32(Imm32(1), output);
13515 0 : masm.jump(&done);
13516 0 : masm.bind(¬Packed);
13517 0 : masm.move32(Imm32(0), output);
13518 :
13519 0 : masm.bind(&done);
13520 0 : }
13521 :
13522 : typedef bool (*GetPrototypeOfFn)(JSContext*, HandleObject, MutableHandleValue);
13523 1 : static const VMFunction GetPrototypeOfInfo =
13524 0 : FunctionInfo<GetPrototypeOfFn>(jit::GetPrototypeOf, "GetPrototypeOf");
13525 :
13526 : void
13527 0 : CodeGenerator::visitGetPrototypeOf(LGetPrototypeOf* lir)
13528 : {
13529 0 : Register target = ToRegister(lir->target());
13530 0 : ValueOperand out = ToOutValue(lir);
13531 0 : Register scratch = out.scratchReg();
13532 :
13533 0 : OutOfLineCode* ool = oolCallVM(GetPrototypeOfInfo, lir, ArgList(target), StoreValueTo(out));
13534 :
13535 0 : MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
13536 :
13537 0 : masm.loadObjProto(target, scratch);
13538 :
13539 0 : Label hasProto;
13540 0 : masm.branchPtr(Assembler::Above, scratch, ImmWord(1), &hasProto);
13541 :
13542 : // Call into the VM for lazy prototypes.
13543 0 : masm.branchPtr(Assembler::Equal, scratch, ImmWord(1), ool->entry());
13544 :
13545 0 : masm.moveValue(NullValue(), out);
13546 0 : masm.jump(ool->rejoin());
13547 :
13548 0 : masm.bind(&hasProto);
13549 0 : masm.tagValue(JSVAL_TYPE_OBJECT, scratch, out);
13550 :
13551 0 : masm.bind(ool->rejoin());
13552 0 : }
13553 :
13554 : static_assert(!std::is_polymorphic<CodeGenerator>::value,
13555 : "CodeGenerator should not have any virtual methods");
13556 :
13557 : } // namespace jit
13558 : } // namespace js
|
