LLVM 20.0.0git
SelectionDAGBuilder.cpp
Go to the documentation of this file.
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements routines for translating from LLVM IR into SelectionDAG IR.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SelectionDAGBuilder.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/BitVector.h"
18#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/StringRef.h"
22#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/Loads.h"
57#include "llvm/IR/Argument.h"
58#include "llvm/IR/Attributes.h"
59#include "llvm/IR/BasicBlock.h"
60#include "llvm/IR/CFG.h"
61#include "llvm/IR/CallingConv.h"
62#include "llvm/IR/Constant.h"
64#include "llvm/IR/Constants.h"
65#include "llvm/IR/DataLayout.h"
66#include "llvm/IR/DebugInfo.h"
71#include "llvm/IR/Function.h"
73#include "llvm/IR/InlineAsm.h"
74#include "llvm/IR/InstrTypes.h"
77#include "llvm/IR/Intrinsics.h"
78#include "llvm/IR/IntrinsicsAArch64.h"
79#include "llvm/IR/IntrinsicsAMDGPU.h"
80#include "llvm/IR/IntrinsicsWebAssembly.h"
81#include "llvm/IR/LLVMContext.h"
83#include "llvm/IR/Metadata.h"
84#include "llvm/IR/Module.h"
85#include "llvm/IR/Operator.h"
87#include "llvm/IR/Statepoint.h"
88#include "llvm/IR/Type.h"
89#include "llvm/IR/User.h"
90#include "llvm/IR/Value.h"
91#include "llvm/MC/MCContext.h"
96#include "llvm/Support/Debug.h"
105#include <cstddef>
106#include <limits>
107#include <optional>
108#include <tuple>
109
110using namespace llvm;
111using namespace PatternMatch;
112using namespace SwitchCG;
113
114#define DEBUG_TYPE "isel"
115
116/// LimitFloatPrecision - Generate low-precision inline sequences for
117/// some float libcalls (6, 8 or 12 bits).
118static unsigned LimitFloatPrecision;
119
120static cl::opt<bool>
121 InsertAssertAlign("insert-assert-align", cl::init(true),
122 cl::desc("Insert the experimental `assertalign` node."),
124
126 LimitFPPrecision("limit-float-precision",
127 cl::desc("Generate low-precision inline sequences "
128 "for some float libcalls"),
130 cl::init(0));
131
133 "switch-peel-threshold", cl::Hidden, cl::init(66),
134 cl::desc("Set the case probability threshold for peeling the case from a "
135 "switch statement. A value greater than 100 will void this "
136 "optimization"));
137
138// Limit the width of DAG chains. This is important in general to prevent
139// DAG-based analysis from blowing up. For example, alias analysis and
140// load clustering may not complete in reasonable time. It is difficult to
141// recognize and avoid this situation within each individual analysis, and
142// future analyses are likely to have the same behavior. Limiting DAG width is
143// the safe approach and will be especially important with global DAGs.
144//
145// MaxParallelChains default is arbitrarily high to avoid affecting
146// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
147// sequence over this should have been converted to llvm.memcpy by the
148// frontend. It is easy to induce this behavior with .ll code such as:
149// %buffer = alloca [4096 x i8]
150// %data = load [4096 x i8]* %argPtr
151// store [4096 x i8] %data, [4096 x i8]* %buffer
152static const unsigned MaxParallelChains = 64;
153
155 const SDValue *Parts, unsigned NumParts,
156 MVT PartVT, EVT ValueVT, const Value *V,
157 SDValue InChain,
158 std::optional<CallingConv::ID> CC);
159
160/// getCopyFromParts - Create a value that contains the specified legal parts
161/// combined into the value they represent. If the parts combine to a type
162/// larger than ValueVT then AssertOp can be used to specify whether the extra
163/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
164/// (ISD::AssertSext).
165static SDValue
166getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts,
167 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V,
168 SDValue InChain,
169 std::optional<CallingConv::ID> CC = std::nullopt,
170 std::optional<ISD::NodeType> AssertOp = std::nullopt) {
171 // Let the target assemble the parts if it wants to
172 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
173 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts,
174 PartVT, ValueVT, CC))
175 return Val;
176
177 if (ValueVT.isVector())
178 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
179 InChain, CC);
180
181 assert(NumParts > 0 && "No parts to assemble!");
182 SDValue Val = Parts[0];
183
184 if (NumParts > 1) {
185 // Assemble the value from multiple parts.
186 if (ValueVT.isInteger()) {
187 unsigned PartBits = PartVT.getSizeInBits();
188 unsigned ValueBits = ValueVT.getSizeInBits();
189
190 // Assemble the power of 2 part.
191 unsigned RoundParts = llvm::bit_floor(NumParts);
192 unsigned RoundBits = PartBits * RoundParts;
193 EVT RoundVT = RoundBits == ValueBits ?
194 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
195 SDValue Lo, Hi;
196
197 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
198
199 if (RoundParts > 2) {
200 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V,
201 InChain);
202 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, RoundParts / 2,
203 PartVT, HalfVT, V, InChain);
204 } else {
205 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
206 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
207 }
208
209 if (DAG.getDataLayout().isBigEndian())
210 std::swap(Lo, Hi);
211
212 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
213
214 if (RoundParts < NumParts) {
215 // Assemble the trailing non-power-of-2 part.
216 unsigned OddParts = NumParts - RoundParts;
217 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
218 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
219 OddVT, V, InChain, CC);
220
221 // Combine the round and odd parts.
222 Lo = Val;
223 if (DAG.getDataLayout().isBigEndian())
224 std::swap(Lo, Hi);
225 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
226 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
227 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
228 DAG.getConstant(Lo.getValueSizeInBits(), DL,
230 TotalVT, DAG.getDataLayout())));
231 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
232 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
233 }
234 } else if (PartVT.isFloatingPoint()) {
235 // FP split into multiple FP parts (for ppcf128)
236 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
237 "Unexpected split");
238 SDValue Lo, Hi;
239 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
240 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
241 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
242 std::swap(Lo, Hi);
243 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
244 } else {
245 // FP split into integer parts (soft fp)
246 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
247 !PartVT.isVector() && "Unexpected split");
248 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
249 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V,
250 InChain, CC);
251 }
252 }
253
254 // There is now one part, held in Val. Correct it to match ValueVT.
255 // PartEVT is the type of the register class that holds the value.
256 // ValueVT is the type of the inline asm operation.
257 EVT PartEVT = Val.getValueType();
258
259 if (PartEVT == ValueVT)
260 return Val;
261
262 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
263 ValueVT.bitsLT(PartEVT)) {
264 // For an FP value in an integer part, we need to truncate to the right
265 // width first.
266 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
267 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
268 }
269
270 // Handle types that have the same size.
271 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
272 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
273
274 // Handle types with different sizes.
275 if (PartEVT.isInteger() && ValueVT.isInteger()) {
276 if (ValueVT.bitsLT(PartEVT)) {
277 // For a truncate, see if we have any information to
278 // indicate whether the truncated bits will always be
279 // zero or sign-extension.
280 if (AssertOp)
281 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
282 DAG.getValueType(ValueVT));
283 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
284 }
285 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
286 }
287
288 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
289 // FP_ROUND's are always exact here.
290 if (ValueVT.bitsLT(Val.getValueType())) {
291
292 SDValue NoChange =
294
296 llvm::Attribute::StrictFP)) {
297 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
298 DAG.getVTList(ValueVT, MVT::Other), InChain, Val,
299 NoChange);
300 }
301
302 return DAG.getNode(ISD::FP_ROUND, DL, ValueVT, Val, NoChange);
303 }
304
305 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
306 }
307
308 // Handle MMX to a narrower integer type by bitcasting MMX to integer and
309 // then truncating.
310 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() &&
311 ValueVT.bitsLT(PartEVT)) {
312 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
313 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
314 }
315
316 report_fatal_error("Unknown mismatch in getCopyFromParts!");
317}
318
320 const Twine &ErrMsg) {
321 const Instruction *I = dyn_cast_or_null<Instruction>(V);
322 if (!I)
323 return Ctx.emitError(ErrMsg);
324
325 if (const CallInst *CI = dyn_cast<CallInst>(I))
326 if (CI->isInlineAsm()) {
328 *CI, ErrMsg + ", possible invalid constraint for vector type"));
329 }
330
331 return Ctx.emitError(I, ErrMsg);
332}
333
334/// getCopyFromPartsVector - Create a value that contains the specified legal
335/// parts combined into the value they represent. If the parts combine to a
336/// type larger than ValueVT then AssertOp can be used to specify whether the
337/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
338/// ValueVT (ISD::AssertSext).
340 const SDValue *Parts, unsigned NumParts,
341 MVT PartVT, EVT ValueVT, const Value *V,
342 SDValue InChain,
343 std::optional<CallingConv::ID> CallConv) {
344 assert(ValueVT.isVector() && "Not a vector value");
345 assert(NumParts > 0 && "No parts to assemble!");
346 const bool IsABIRegCopy = CallConv.has_value();
347
348 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
349 SDValue Val = Parts[0];
350
351 // Handle a multi-element vector.
352 if (NumParts > 1) {
353 EVT IntermediateVT;
354 MVT RegisterVT;
355 unsigned NumIntermediates;
356 unsigned NumRegs;
357
358 if (IsABIRegCopy) {
360 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT,
361 NumIntermediates, RegisterVT);
362 } else {
363 NumRegs =
364 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
365 NumIntermediates, RegisterVT);
366 }
367
368 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
369 NumParts = NumRegs; // Silence a compiler warning.
370 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
371 assert(RegisterVT.getSizeInBits() ==
372 Parts[0].getSimpleValueType().getSizeInBits() &&
373 "Part type sizes don't match!");
374
375 // Assemble the parts into intermediate operands.
376 SmallVector<SDValue, 8> Ops(NumIntermediates);
377 if (NumIntermediates == NumParts) {
378 // If the register was not expanded, truncate or copy the value,
379 // as appropriate.
380 for (unsigned i = 0; i != NumParts; ++i)
381 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, PartVT, IntermediateVT,
382 V, InChain, CallConv);
383 } else if (NumParts > 0) {
384 // If the intermediate type was expanded, build the intermediate
385 // operands from the parts.
386 assert(NumParts % NumIntermediates == 0 &&
387 "Must expand into a divisible number of parts!");
388 unsigned Factor = NumParts / NumIntermediates;
389 for (unsigned i = 0; i != NumIntermediates; ++i)
390 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, PartVT,
391 IntermediateVT, V, InChain, CallConv);
392 }
393
394 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
395 // intermediate operands.
396 EVT BuiltVectorTy =
397 IntermediateVT.isVector()
399 *DAG.getContext(), IntermediateVT.getScalarType(),
400 IntermediateVT.getVectorElementCount() * NumParts)
402 IntermediateVT.getScalarType(),
403 NumIntermediates);
404 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
406 DL, BuiltVectorTy, Ops);
407 }
408
409 // There is now one part, held in Val. Correct it to match ValueVT.
410 EVT PartEVT = Val.getValueType();
411
412 if (PartEVT == ValueVT)
413 return Val;
414
415 if (PartEVT.isVector()) {
416 // Vector/Vector bitcast.
417 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
418 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
419
420 // If the parts vector has more elements than the value vector, then we
421 // have a vector widening case (e.g. <2 x float> -> <4 x float>).
422 // Extract the elements we want.
423 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) {
426 (PartEVT.getVectorElementCount().isScalable() ==
427 ValueVT.getVectorElementCount().isScalable()) &&
428 "Cannot narrow, it would be a lossy transformation");
429 PartEVT =
431 ValueVT.getVectorElementCount());
432 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val,
433 DAG.getVectorIdxConstant(0, DL));
434 if (PartEVT == ValueVT)
435 return Val;
436 if (PartEVT.isInteger() && ValueVT.isFloatingPoint())
437 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
438
439 // Vector/Vector bitcast (e.g. <2 x bfloat> -> <2 x half>).
440 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
441 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
442 }
443
444 // Promoted vector extract
445 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
446 }
447
448 // Trivial bitcast if the types are the same size and the destination
449 // vector type is legal.
450 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
451 TLI.isTypeLegal(ValueVT))
452 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
453
454 if (ValueVT.getVectorNumElements() != 1) {
455 // Certain ABIs require that vectors are passed as integers. For vectors
456 // are the same size, this is an obvious bitcast.
457 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
458 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
459 } else if (ValueVT.bitsLT(PartEVT)) {
460 const uint64_t ValueSize = ValueVT.getFixedSizeInBits();
461 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
462 // Drop the extra bits.
463 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
464 return DAG.getBitcast(ValueVT, Val);
465 }
466
468 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
469 return DAG.getUNDEF(ValueVT);
470 }
471
472 // Handle cases such as i8 -> <1 x i1>
473 EVT ValueSVT = ValueVT.getVectorElementType();
474 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) {
475 unsigned ValueSize = ValueSVT.getSizeInBits();
476 if (ValueSize == PartEVT.getSizeInBits()) {
477 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val);
478 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) {
479 // It's possible a scalar floating point type gets softened to integer and
480 // then promoted to a larger integer. If PartEVT is the larger integer
481 // we need to truncate it and then bitcast to the FP type.
482 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types");
483 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
484 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val);
485 Val = DAG.getBitcast(ValueSVT, Val);
486 } else {
487 Val = ValueVT.isFloatingPoint()
488 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
489 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
490 }
491 }
492
493 return DAG.getBuildVector(ValueVT, DL, Val);
494}
495
496static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
497 SDValue Val, SDValue *Parts, unsigned NumParts,
498 MVT PartVT, const Value *V,
499 std::optional<CallingConv::ID> CallConv);
500
501/// getCopyToParts - Create a series of nodes that contain the specified value
502/// split into legal parts. If the parts contain more bits than Val, then, for
503/// integers, ExtendKind can be used to specify how to generate the extra bits.
504static void
506 unsigned NumParts, MVT PartVT, const Value *V,
507 std::optional<CallingConv::ID> CallConv = std::nullopt,
508 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
509 // Let the target split the parts if it wants to
510 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
511 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT,
512 CallConv))
513 return;
514 EVT ValueVT = Val.getValueType();
515
516 // Handle the vector case separately.
517 if (ValueVT.isVector())
518 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
519 CallConv);
520
521 unsigned OrigNumParts = NumParts;
523 "Copying to an illegal type!");
524
525 if (NumParts == 0)
526 return;
527
528 assert(!ValueVT.isVector() && "Vector case handled elsewhere");
529 EVT PartEVT = PartVT;
530 if (PartEVT == ValueVT) {
531 assert(NumParts == 1 && "No-op copy with multiple parts!");
532 Parts[0] = Val;
533 return;
534 }
535
536 unsigned PartBits = PartVT.getSizeInBits();
537 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
538 // If the parts cover more bits than the value has, promote the value.
539 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
540 assert(NumParts == 1 && "Do not know what to promote to!");
541 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
542 } else {
543 if (ValueVT.isFloatingPoint()) {
544 // FP values need to be bitcast, then extended if they are being put
545 // into a larger container.
546 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
547 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
548 }
549 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
550 ValueVT.isInteger() &&
551 "Unknown mismatch!");
552 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
553 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
554 if (PartVT == MVT::x86mmx)
555 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
556 }
557 } else if (PartBits == ValueVT.getSizeInBits()) {
558 // Different types of the same size.
559 assert(NumParts == 1 && PartEVT != ValueVT);
560 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
561 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
562 // If the parts cover less bits than value has, truncate the value.
563 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
564 ValueVT.isInteger() &&
565 "Unknown mismatch!");
566 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
567 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
568 if (PartVT == MVT::x86mmx)
569 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
570 }
571
572 // The value may have changed - recompute ValueVT.
573 ValueVT = Val.getValueType();
574 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
575 "Failed to tile the value with PartVT!");
576
577 if (NumParts == 1) {
578 if (PartEVT != ValueVT) {
580 "scalar-to-vector conversion failed");
581 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
582 }
583
584 Parts[0] = Val;
585 return;
586 }
587
588 // Expand the value into multiple parts.
589 if (NumParts & (NumParts - 1)) {
590 // The number of parts is not a power of 2. Split off and copy the tail.
591 assert(PartVT.isInteger() && ValueVT.isInteger() &&
592 "Do not know what to expand to!");
593 unsigned RoundParts = llvm::bit_floor(NumParts);
594 unsigned RoundBits = RoundParts * PartBits;
595 unsigned OddParts = NumParts - RoundParts;
596 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
597 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL));
598
599 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
600 CallConv);
601
602 if (DAG.getDataLayout().isBigEndian())
603 // The odd parts were reversed by getCopyToParts - unreverse them.
604 std::reverse(Parts + RoundParts, Parts + NumParts);
605
606 NumParts = RoundParts;
607 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
608 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
609 }
610
611 // The number of parts is a power of 2. Repeatedly bisect the value using
612 // EXTRACT_ELEMENT.
613 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
615 ValueVT.getSizeInBits()),
616 Val);
617
618 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
619 for (unsigned i = 0; i < NumParts; i += StepSize) {
620 unsigned ThisBits = StepSize * PartBits / 2;
621 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
622 SDValue &Part0 = Parts[i];
623 SDValue &Part1 = Parts[i+StepSize/2];
624
625 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
626 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
627 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
628 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
629
630 if (ThisBits == PartBits && ThisVT != PartVT) {
631 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
632 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
633 }
634 }
635 }
636
637 if (DAG.getDataLayout().isBigEndian())
638 std::reverse(Parts, Parts + OrigNumParts);
639}
640
642 const SDLoc &DL, EVT PartVT) {
643 if (!PartVT.isVector())
644 return SDValue();
645
646 EVT ValueVT = Val.getValueType();
647 EVT PartEVT = PartVT.getVectorElementType();
648 EVT ValueEVT = ValueVT.getVectorElementType();
649 ElementCount PartNumElts = PartVT.getVectorElementCount();
650 ElementCount ValueNumElts = ValueVT.getVectorElementCount();
651
652 // We only support widening vectors with equivalent element types and
653 // fixed/scalable properties. If a target needs to widen a fixed-length type
654 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below.
655 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) ||
656 PartNumElts.isScalable() != ValueNumElts.isScalable())
657 return SDValue();
658
659 // Have a try for bf16 because some targets share its ABI with fp16.
660 if (ValueEVT == MVT::bf16 && PartEVT == MVT::f16) {
662 "Cannot widen to illegal type");
663 Val = DAG.getNode(ISD::BITCAST, DL,
664 ValueVT.changeVectorElementType(MVT::f16), Val);
665 } else if (PartEVT != ValueEVT) {
666 return SDValue();
667 }
668
669 // Widening a scalable vector to another scalable vector is done by inserting
670 // the vector into a larger undef one.
671 if (PartNumElts.isScalable())
672 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
673 Val, DAG.getVectorIdxConstant(0, DL));
674
675 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
676 // undef elements.
678 DAG.ExtractVectorElements(Val, Ops);
679 SDValue EltUndef = DAG.getUNDEF(PartEVT);
680 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef);
681
682 // FIXME: Use CONCAT for 2x -> 4x.
683 return DAG.getBuildVector(PartVT, DL, Ops);
684}
685
686/// getCopyToPartsVector - Create a series of nodes that contain the specified
687/// value split into legal parts.
688static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
689 SDValue Val, SDValue *Parts, unsigned NumParts,
690 MVT PartVT, const Value *V,
691 std::optional<CallingConv::ID> CallConv) {
692 EVT ValueVT = Val.getValueType();
693 assert(ValueVT.isVector() && "Not a vector");
694 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
695 const bool IsABIRegCopy = CallConv.has_value();
696
697 if (NumParts == 1) {
698 EVT PartEVT = PartVT;
699 if (PartEVT == ValueVT) {
700 // Nothing to do.
701 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
702 // Bitconvert vector->vector case.
703 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
704 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
705 Val = Widened;
706 } else if (PartVT.isVector() &&
708 ValueVT.getVectorElementType()) &&
709 PartEVT.getVectorElementCount() ==
710 ValueVT.getVectorElementCount()) {
711
712 // Promoted vector extract
713 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
714 } else if (PartEVT.isVector() &&
715 PartEVT.getVectorElementType() !=
716 ValueVT.getVectorElementType() &&
717 TLI.getTypeAction(*DAG.getContext(), ValueVT) ==
718 TargetLowering::TypeWidenVector) {
719 // Combination of widening and promotion.
720 EVT WidenVT =
722 PartVT.getVectorElementCount());
723 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT);
724 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT);
725 } else {
726 // Don't extract an integer from a float vector. This can happen if the
727 // FP type gets softened to integer and then promoted. The promotion
728 // prevents it from being picked up by the earlier bitcast case.
729 if (ValueVT.getVectorElementCount().isScalar() &&
730 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) {
731 // If we reach this condition and PartVT is FP, this means that
732 // ValueVT is also FP and both have a different size, otherwise we
733 // would have bitcasted them. Producing an EXTRACT_VECTOR_ELT here
734 // would be invalid since that would mean the smaller FP type has to
735 // be extended to the larger one.
736 if (PartVT.isFloatingPoint()) {
737 Val = DAG.getBitcast(ValueVT.getScalarType(), Val);
738 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
739 } else
740 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
741 DAG.getVectorIdxConstant(0, DL));
742 } else {
743 uint64_t ValueSize = ValueVT.getFixedSizeInBits();
744 assert(PartVT.getFixedSizeInBits() > ValueSize &&
745 "lossy conversion of vector to scalar type");
746 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize);
747 Val = DAG.getBitcast(IntermediateType, Val);
748 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
749 }
750 }
751
752 assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
753 Parts[0] = Val;
754 return;
755 }
756
757 // Handle a multi-element vector.
758 EVT IntermediateVT;
759 MVT RegisterVT;
760 unsigned NumIntermediates;
761 unsigned NumRegs;
762 if (IsABIRegCopy) {
764 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates,
765 RegisterVT);
766 } else {
767 NumRegs =
768 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
769 NumIntermediates, RegisterVT);
770 }
771
772 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
773 NumParts = NumRegs; // Silence a compiler warning.
774 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
775
776 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() &&
777 "Mixing scalable and fixed vectors when copying in parts");
778
779 std::optional<ElementCount> DestEltCnt;
780
781 if (IntermediateVT.isVector())
782 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates;
783 else
784 DestEltCnt = ElementCount::getFixed(NumIntermediates);
785
786 EVT BuiltVectorTy = EVT::getVectorVT(
787 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt);
788
789 if (ValueVT == BuiltVectorTy) {
790 // Nothing to do.
791 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) {
792 // Bitconvert vector->vector case.
793 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
794 } else {
795 if (BuiltVectorTy.getVectorElementType().bitsGT(
796 ValueVT.getVectorElementType())) {
797 // Integer promotion.
798 ValueVT = EVT::getVectorVT(*DAG.getContext(),
799 BuiltVectorTy.getVectorElementType(),
800 ValueVT.getVectorElementCount());
801 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
802 }
803
804 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) {
805 Val = Widened;
806 }
807 }
808
809 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type");
810
811 // Split the vector into intermediate operands.
812 SmallVector<SDValue, 8> Ops(NumIntermediates);
813 for (unsigned i = 0; i != NumIntermediates; ++i) {
814 if (IntermediateVT.isVector()) {
815 // This does something sensible for scalable vectors - see the
816 // definition of EXTRACT_SUBVECTOR for further details.
817 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements();
818 Ops[i] =
819 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
820 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL));
821 } else {
822 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
823 DAG.getVectorIdxConstant(i, DL));
824 }
825 }
826
827 // Split the intermediate operands into legal parts.
828 if (NumParts == NumIntermediates) {
829 // If the register was not expanded, promote or copy the value,
830 // as appropriate.
831 for (unsigned i = 0; i != NumParts; ++i)
832 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
833 } else if (NumParts > 0) {
834 // If the intermediate type was expanded, split each the value into
835 // legal parts.
836 assert(NumIntermediates != 0 && "division by zero");
837 assert(NumParts % NumIntermediates == 0 &&
838 "Must expand into a divisible number of parts!");
839 unsigned Factor = NumParts / NumIntermediates;
840 for (unsigned i = 0; i != NumIntermediates; ++i)
841 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
842 CallConv);
843 }
844}
845
847 EVT valuevt, std::optional<CallingConv::ID> CC)
848 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
849 RegCount(1, regs.size()), CallConv(CC) {}
850
852 const DataLayout &DL, Register Reg, Type *Ty,
853 std::optional<CallingConv::ID> CC) {
854 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
855
856 CallConv = CC;
857
858 for (EVT ValueVT : ValueVTs) {
859 unsigned NumRegs =
861 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT)
862 : TLI.getNumRegisters(Context, ValueVT);
863 MVT RegisterVT =
865 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT)
866 : TLI.getRegisterType(Context, ValueVT);
867 for (unsigned i = 0; i != NumRegs; ++i)
868 Regs.push_back(Reg + i);
869 RegVTs.push_back(RegisterVT);
870 RegCount.push_back(NumRegs);
871 Reg = Reg.id() + NumRegs;
872 }
873}
874
876 FunctionLoweringInfo &FuncInfo,
877 const SDLoc &dl, SDValue &Chain,
878 SDValue *Glue, const Value *V) const {
879 // A Value with type {} or [0 x %t] needs no registers.
880 if (ValueVTs.empty())
881 return SDValue();
882
883 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
884
885 // Assemble the legal parts into the final values.
886 SmallVector<SDValue, 4> Values(ValueVTs.size());
888 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
889 // Copy the legal parts from the registers.
890 EVT ValueVT = ValueVTs[Value];
891 unsigned NumRegs = RegCount[Value];
892 MVT RegisterVT = isABIMangled()
894 *DAG.getContext(), *CallConv, RegVTs[Value])
895 : RegVTs[Value];
896
897 Parts.resize(NumRegs);
898 for (unsigned i = 0; i != NumRegs; ++i) {
899 SDValue P;
900 if (!Glue) {
901 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
902 } else {
903 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue);
904 *Glue = P.getValue(2);
905 }
906
907 Chain = P.getValue(1);
908 Parts[i] = P;
909
910 // If the source register was virtual and if we know something about it,
911 // add an assert node.
912 if (!Register::isVirtualRegister(Regs[Part + i]) ||
913 !RegisterVT.isInteger())
914 continue;
915
917 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
918 if (!LOI)
919 continue;
920
921 unsigned RegSize = RegisterVT.getScalarSizeInBits();
922 unsigned NumSignBits = LOI->NumSignBits;
923 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
924
925 if (NumZeroBits == RegSize) {
926 // The current value is a zero.
927 // Explicitly express that as it would be easier for
928 // optimizations to kick in.
929 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
930 continue;
931 }
932
933 // FIXME: We capture more information than the dag can represent. For
934 // now, just use the tightest assertzext/assertsext possible.
935 bool isSExt;
936 EVT FromVT(MVT::Other);
937 if (NumZeroBits) {
938 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
939 isSExt = false;
940 } else if (NumSignBits > 1) {
941 FromVT =
942 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
943 isSExt = true;
944 } else {
945 continue;
946 }
947 // Add an assertion node.
948 assert(FromVT != MVT::Other);
949 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
950 RegisterVT, P, DAG.getValueType(FromVT));
951 }
952
953 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
954 RegisterVT, ValueVT, V, Chain, CallConv);
955 Part += NumRegs;
956 Parts.clear();
957 }
958
959 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
960}
961
963 const SDLoc &dl, SDValue &Chain, SDValue *Glue,
964 const Value *V,
965 ISD::NodeType PreferredExtendType) const {
966 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
967 ISD::NodeType ExtendKind = PreferredExtendType;
968
969 // Get the list of the values's legal parts.
970 unsigned NumRegs = Regs.size();
971 SmallVector<SDValue, 8> Parts(NumRegs);
972 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
973 unsigned NumParts = RegCount[Value];
974
975 MVT RegisterVT = isABIMangled()
977 *DAG.getContext(), *CallConv, RegVTs[Value])
978 : RegVTs[Value];
979
980 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
981 ExtendKind = ISD::ZERO_EXTEND;
982
983 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
984 NumParts, RegisterVT, V, CallConv, ExtendKind);
985 Part += NumParts;
986 }
987
988 // Copy the parts into the registers.
989 SmallVector<SDValue, 8> Chains(NumRegs);
990 for (unsigned i = 0; i != NumRegs; ++i) {
991 SDValue Part;
992 if (!Glue) {
993 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
994 } else {
995 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Glue);
996 *Glue = Part.getValue(1);
997 }
998
999 Chains[i] = Part.getValue(0);
1000 }
1001
1002 if (NumRegs == 1 || Glue)
1003 // If NumRegs > 1 && Glue is used then the use of the last CopyToReg is
1004 // flagged to it. That is the CopyToReg nodes and the user are considered
1005 // a single scheduling unit. If we create a TokenFactor and return it as
1006 // chain, then the TokenFactor is both a predecessor (operand) of the
1007 // user as well as a successor (the TF operands are flagged to the user).
1008 // c1, f1 = CopyToReg
1009 // c2, f2 = CopyToReg
1010 // c3 = TokenFactor c1, c2
1011 // ...
1012 // = op c3, ..., f2
1013 Chain = Chains[NumRegs-1];
1014 else
1015 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1016}
1017
1019 unsigned MatchingIdx, const SDLoc &dl,
1020 SelectionDAG &DAG,
1021 std::vector<SDValue> &Ops) const {
1022 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1023
1024 InlineAsm::Flag Flag(Code, Regs.size());
1025 if (HasMatching)
1026 Flag.setMatchingOp(MatchingIdx);
1027 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) {
1028 // Put the register class of the virtual registers in the flag word. That
1029 // way, later passes can recompute register class constraints for inline
1030 // assembly as well as normal instructions.
1031 // Don't do this for tied operands that can use the regclass information
1032 // from the def.
1034 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
1035 Flag.setRegClass(RC->getID());
1036 }
1037
1038 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
1039 Ops.push_back(Res);
1040
1041 if (Code == InlineAsm::Kind::Clobber) {
1042 // Clobbers should always have a 1:1 mapping with registers, and may
1043 // reference registers that have illegal (e.g. vector) types. Hence, we
1044 // shouldn't try to apply any sort of splitting logic to them.
1045 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
1046 "No 1:1 mapping from clobbers to regs?");
1048 (void)SP;
1049 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
1050 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
1051 assert(
1052 (Regs[I] != SP ||
1054 "If we clobbered the stack pointer, MFI should know about it.");
1055 }
1056 return;
1057 }
1058
1059 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
1060 MVT RegisterVT = RegVTs[Value];
1061 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value],
1062 RegisterVT);
1063 for (unsigned i = 0; i != NumRegs; ++i) {
1064 assert(Reg < Regs.size() && "Mismatch in # registers expected");
1065 unsigned TheReg = Regs[Reg++];
1066 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
1067 }
1068 }
1069}
1070
1074 unsigned I = 0;
1075 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1076 unsigned RegCount = std::get<0>(CountAndVT);
1077 MVT RegisterVT = std::get<1>(CountAndVT);
1078 TypeSize RegisterSize = RegisterVT.getSizeInBits();
1079 for (unsigned E = I + RegCount; I != E; ++I)
1080 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1081 }
1082 return OutVec;
1083}
1084
1086 AssumptionCache *ac,
1087 const TargetLibraryInfo *li) {
1088 AA = aa;
1089 AC = ac;
1090 GFI = gfi;
1091 LibInfo = li;
1093 LPadToCallSiteMap.clear();
1095 AssignmentTrackingEnabled = isAssignmentTrackingEnabled(
1097}
1098
1100 NodeMap.clear();
1101 UnusedArgNodeMap.clear();
1102 PendingLoads.clear();
1103 PendingExports.clear();
1104 PendingConstrainedFP.clear();
1105 PendingConstrainedFPStrict.clear();
1106 CurInst = nullptr;
1107 HasTailCall = false;
1108 SDNodeOrder = LowestSDNodeOrder;
1110}
1111
1113 DanglingDebugInfoMap.clear();
1114}
1115
1116// Update DAG root to include dependencies on Pending chains.
1117SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) {
1118 SDValue Root = DAG.getRoot();
1119
1120 if (Pending.empty())
1121 return Root;
1122
1123 // Add current root to PendingChains, unless we already indirectly
1124 // depend on it.
1125 if (Root.getOpcode() != ISD::EntryToken) {
1126 unsigned i = 0, e = Pending.size();
1127 for (; i != e; ++i) {
1128 assert(Pending[i].getNode()->getNumOperands() > 1);
1129 if (Pending[i].getNode()->getOperand(0) == Root)
1130 break; // Don't add the root if we already indirectly depend on it.
1131 }
1132
1133 if (i == e)
1134 Pending.push_back(Root);
1135 }
1136
1137 if (Pending.size() == 1)
1138 Root = Pending[0];
1139 else
1140 Root = DAG.getTokenFactor(getCurSDLoc(), Pending);
1141
1142 DAG.setRoot(Root);
1143 Pending.clear();
1144 return Root;
1145}
1146
1148 return updateRoot(PendingLoads);
1149}
1150
1152 // Chain up all pending constrained intrinsics together with all
1153 // pending loads, by simply appending them to PendingLoads and
1154 // then calling getMemoryRoot().
1155 PendingLoads.reserve(PendingLoads.size() +
1156 PendingConstrainedFP.size() +
1157 PendingConstrainedFPStrict.size());
1158 PendingLoads.append(PendingConstrainedFP.begin(),
1159 PendingConstrainedFP.end());
1160 PendingLoads.append(PendingConstrainedFPStrict.begin(),
1161 PendingConstrainedFPStrict.end());
1162 PendingConstrainedFP.clear();
1163 PendingConstrainedFPStrict.clear();
1164 return getMemoryRoot();
1165}
1166
1168 // We need to emit pending fpexcept.strict constrained intrinsics,
1169 // so append them to the PendingExports list.
1170 PendingExports.append(PendingConstrainedFPStrict.begin(),
1171 PendingConstrainedFPStrict.end());
1172 PendingConstrainedFPStrict.clear();
1173 return updateRoot(PendingExports);
1174}
1175
1177 DILocalVariable *Variable,
1179 DebugLoc DL) {
1180 assert(Variable && "Missing variable");
1181
1182 // Check if address has undef value.
1183 if (!Address || isa<UndefValue>(Address) ||
1184 (Address->use_empty() && !isa<Argument>(Address))) {
1185 LLVM_DEBUG(
1186 dbgs()
1187 << "dbg_declare: Dropping debug info (bad/undef/unused-arg address)\n");
1188 return;
1189 }
1190
1191 bool IsParameter = Variable->isParameter() || isa<Argument>(Address);
1192
1193 SDValue &N = NodeMap[Address];
1194 if (!N.getNode() && isa<Argument>(Address))
1195 // Check unused arguments map.
1196 N = UnusedArgNodeMap[Address];
1197 SDDbgValue *SDV;
1198 if (N.getNode()) {
1199 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
1200 Address = BCI->getOperand(0);
1201 // Parameters are handled specially.
1202 auto *FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
1203 if (IsParameter && FINode) {
1204 // Byval parameter. We have a frame index at this point.
1205 SDV = DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
1206 /*IsIndirect*/ true, DL, SDNodeOrder);
1207 } else if (isa<Argument>(Address)) {
1208 // Address is an argument, so try to emit its dbg value using
1209 // virtual register info from the FuncInfo.ValueMap.
1210 EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1211 FuncArgumentDbgValueKind::Declare, N);
1212 return;
1213 } else {
1214 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
1215 true, DL, SDNodeOrder);
1216 }
1217 DAG.AddDbgValue(SDV, IsParameter);
1218 } else {
1219 // If Address is an argument then try to emit its dbg value using
1220 // virtual register info from the FuncInfo.ValueMap.
1221 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, DL,
1222 FuncArgumentDbgValueKind::Declare, N)) {
1223 LLVM_DEBUG(dbgs() << "dbg_declare: Dropping debug info"
1224 << " (could not emit func-arg dbg_value)\n");
1225 }
1226 }
1227}
1228
1230 // Add SDDbgValue nodes for any var locs here. Do so before updating
1231 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1232 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) {
1233 // Add SDDbgValue nodes for any var locs here. Do so before updating
1234 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}.
1235 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I);
1236 It != End; ++It) {
1237 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID);
1238 dropDanglingDebugInfo(Var, It->Expr);
1239 if (It->Values.isKillLocation(It->Expr)) {
1240 handleKillDebugValue(Var, It->Expr, It->DL, SDNodeOrder);
1241 continue;
1242 }
1243 SmallVector<Value *> Values(It->Values.location_ops());
1244 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder,
1245 It->Values.hasArgList())) {
1246 SmallVector<Value *, 4> Vals(It->Values.location_ops());
1248 FnVarLocs->getDILocalVariable(It->VariableID),
1249 It->Expr, Vals.size() > 1, It->DL, SDNodeOrder);
1250 }
1251 }
1252 }
1253
1254 // We must skip DbgVariableRecords if they've already been processed above as
1255 // we have just emitted the debug values resulting from assignment tracking
1256 // analysis, making any existing DbgVariableRecords redundant (and probably
1257 // less correct). We still need to process DbgLabelRecords. This does sink
1258 // DbgLabelRecords to the bottom of the group of debug records. That sholdn't
1259 // be important as it does so deterministcally and ordering between
1260 // DbgLabelRecords and DbgVariableRecords is immaterial (other than for MIR/IR
1261 // printing).
1262 bool SkipDbgVariableRecords = DAG.getFunctionVarLocs();
1263 // Is there is any debug-info attached to this instruction, in the form of
1264 // DbgRecord non-instruction debug-info records.
1265 for (DbgRecord &DR : I.getDbgRecordRange()) {
1266 if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {
1267 assert(DLR->getLabel() && "Missing label");
1268 SDDbgLabel *SDV =
1269 DAG.getDbgLabel(DLR->getLabel(), DLR->getDebugLoc(), SDNodeOrder);
1270 DAG.AddDbgLabel(SDV);
1271 continue;
1272 }
1273
1274 if (SkipDbgVariableRecords)
1275 continue;
1276 DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
1277 DILocalVariable *Variable = DVR.getVariable();
1280
1282 if (FuncInfo.PreprocessedDVRDeclares.contains(&DVR))
1283 continue;
1284 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DVR
1285 << "\n");
1287 DVR.getDebugLoc());
1288 continue;
1289 }
1290
1291 // A DbgVariableRecord with no locations is a kill location.
1293 if (Values.empty()) {
1295 SDNodeOrder);
1296 continue;
1297 }
1298
1299 // A DbgVariableRecord with an undef or absent location is also a kill
1300 // location.
1301 if (llvm::any_of(Values,
1302 [](Value *V) { return !V || isa<UndefValue>(V); })) {
1304 SDNodeOrder);
1305 continue;
1306 }
1307
1308 bool IsVariadic = DVR.hasArgList();
1309 if (!handleDebugValue(Values, Variable, Expression, DVR.getDebugLoc(),
1310 SDNodeOrder, IsVariadic)) {
1311 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
1312 DVR.getDebugLoc(), SDNodeOrder);
1313 }
1314 }
1315}
1316
1318 visitDbgInfo(I);
1319
1320 // Set up outgoing PHI node register values before emitting the terminator.
1321 if (I.isTerminator()) {
1322 HandlePHINodesInSuccessorBlocks(I.getParent());
1323 }
1324
1325 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1326 if (!isa<DbgInfoIntrinsic>(I))
1327 ++SDNodeOrder;
1328
1329 CurInst = &I;
1330
1331 // Set inserted listener only if required.
1332 bool NodeInserted = false;
1333 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener;
1334 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections);
1335 MDNode *MMRA = I.getMetadata(LLVMContext::MD_mmra);
1336 if (PCSectionsMD || MMRA) {
1337 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>(
1338 DAG, [&](SDNode *) { NodeInserted = true; });
1339 }
1340
1341 visit(I.getOpcode(), I);
1342
1343 if (!I.isTerminator() && !HasTailCall &&
1344 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally
1346
1347 // Handle metadata.
1348 if (PCSectionsMD || MMRA) {
1349 auto It = NodeMap.find(&I);
1350 if (It != NodeMap.end()) {
1351 if (PCSectionsMD)
1352 DAG.addPCSections(It->second.getNode(), PCSectionsMD);
1353 if (MMRA)
1354 DAG.addMMRAMetadata(It->second.getNode(), MMRA);
1355 } else if (NodeInserted) {
1356 // This should not happen; if it does, don't let it go unnoticed so we can
1357 // fix it. Relevant visit*() function is probably missing a setValue().
1358 errs() << "warning: loosing !pcsections and/or !mmra metadata ["
1359 << I.getModule()->getName() << "]\n";
1360 LLVM_DEBUG(I.dump());
1361 assert(false);
1362 }
1363 }
1364
1365 CurInst = nullptr;
1366}
1367
1368void SelectionDAGBuilder::visitPHI(const PHINode &) {
1369 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1370}
1371
1372void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1373 // Note: this doesn't use InstVisitor, because it has to work with
1374 // ConstantExpr's in addition to instructions.
1375 switch (Opcode) {
1376 default: llvm_unreachable("Unknown instruction type encountered!");
1377 // Build the switch statement using the Instruction.def file.
1378#define HANDLE_INST(NUM, OPCODE, CLASS) \
1379 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1380#include "llvm/IR/Instruction.def"
1381 }
1382}
1383
1385 DILocalVariable *Variable,
1386 DebugLoc DL, unsigned Order,
1389 // For variadic dbg_values we will now insert an undef.
1390 // FIXME: We can potentially recover these!
1392 for (const Value *V : Values) {
1393 auto *Undef = UndefValue::get(V->getType());
1395 }
1396 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {},
1397 /*IsIndirect=*/false, DL, Order,
1398 /*IsVariadic=*/true);
1399 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1400 return true;
1401}
1402
1404 DILocalVariable *Var,
1405 DIExpression *Expr,
1406 bool IsVariadic, DebugLoc DL,
1407 unsigned Order) {
1408 if (IsVariadic) {
1409 handleDanglingVariadicDebugInfo(DAG, Var, DL, Order, Values, Expr);
1410 return;
1411 }
1412 // TODO: Dangling debug info will eventually either be resolved or produce
1413 // an Undef DBG_VALUE. However in the resolution case, a gap may appear
1414 // between the original dbg.value location and its resolved DBG_VALUE,
1415 // which we should ideally fill with an extra Undef DBG_VALUE.
1416 assert(Values.size() == 1);
1417 DanglingDebugInfoMap[Values[0]].emplace_back(Var, Expr, DL, Order);
1418}
1419
1421 const DIExpression *Expr) {
1422 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1423 DIVariable *DanglingVariable = DDI.getVariable();
1424 DIExpression *DanglingExpr = DDI.getExpression();
1425 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1426 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for "
1427 << printDDI(nullptr, DDI) << "\n");
1428 return true;
1429 }
1430 return false;
1431 };
1432
1433 for (auto &DDIMI : DanglingDebugInfoMap) {
1434 DanglingDebugInfoVector &DDIV = DDIMI.second;
1435
1436 // If debug info is to be dropped, run it through final checks to see
1437 // whether it can be salvaged.
1438 for (auto &DDI : DDIV)
1439 if (isMatchingDbgValue(DDI))
1440 salvageUnresolvedDbgValue(DDIMI.first, DDI);
1441
1442 erase_if(DDIV, isMatchingDbgValue);
1443 }
1444}
1445
1446// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1447// generate the debug data structures now that we've seen its definition.
1449 SDValue Val) {
1450 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1451 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1452 return;
1453
1454 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1455 for (auto &DDI : DDIV) {
1456 DebugLoc DL = DDI.getDebugLoc();
1457 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1458 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1459 DILocalVariable *Variable = DDI.getVariable();
1460 DIExpression *Expr = DDI.getExpression();
1462 "Expected inlined-at fields to agree");
1463 SDDbgValue *SDV;
1464 if (Val.getNode()) {
1465 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1466 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1467 // we couldn't resolve it directly when examining the DbgValue intrinsic
1468 // in the first place we should not be more successful here). Unless we
1469 // have some test case that prove this to be correct we should avoid
1470 // calling EmitFuncArgumentDbgValue here.
1471 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL,
1472 FuncArgumentDbgValueKind::Value, Val)) {
1473 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for "
1474 << printDDI(V, DDI) << "\n");
1475 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump());
1476 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1477 // inserted after the definition of Val when emitting the instructions
1478 // after ISel. An alternative could be to teach
1479 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1480 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1481 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1482 << ValSDNodeOrder << "\n");
1483 SDV = getDbgValue(Val, Variable, Expr, DL,
1484 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1485 DAG.AddDbgValue(SDV, false);
1486 } else
1487 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for "
1488 << printDDI(V, DDI)
1489 << " in EmitFuncArgumentDbgValue\n");
1490 } else {
1491 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(V, DDI)
1492 << "\n");
1493 auto Undef = UndefValue::get(V->getType());
1494 auto SDV =
1495 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder);
1496 DAG.AddDbgValue(SDV, false);
1497 }
1498 }
1499 DDIV.clear();
1500}
1501
1503 DanglingDebugInfo &DDI) {
1504 // TODO: For the variadic implementation, instead of only checking the fail
1505 // state of `handleDebugValue`, we need know specifically which values were
1506 // invalid, so that we attempt to salvage only those values when processing
1507 // a DIArgList.
1508 const Value *OrigV = V;
1509 DILocalVariable *Var = DDI.getVariable();
1510 DIExpression *Expr = DDI.getExpression();
1511 DebugLoc DL = DDI.getDebugLoc();
1512 unsigned SDOrder = DDI.getSDNodeOrder();
1513
1514 // Currently we consider only dbg.value intrinsics -- we tell the salvager
1515 // that DW_OP_stack_value is desired.
1516 bool StackValue = true;
1517
1518 // Can this Value can be encoded without any further work?
1519 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false))
1520 return;
1521
1522 // Attempt to salvage back through as many instructions as possible. Bail if
1523 // a non-instruction is seen, such as a constant expression or global
1524 // variable. FIXME: Further work could recover those too.
1525 while (isa<Instruction>(V)) {
1526 const Instruction &VAsInst = *cast<const Instruction>(V);
1527 // Temporary "0", awaiting real implementation.
1529 SmallVector<Value *, 4> AdditionalValues;
1530 V = salvageDebugInfoImpl(const_cast<Instruction &>(VAsInst),
1531 Expr->getNumLocationOperands(), Ops,
1532 AdditionalValues);
1533 // If we cannot salvage any further, and haven't yet found a suitable debug
1534 // expression, bail out.
1535 if (!V)
1536 break;
1537
1538 // TODO: If AdditionalValues isn't empty, then the salvage can only be
1539 // represented with a DBG_VALUE_LIST, so we give up. When we have support
1540 // here for variadic dbg_values, remove that condition.
1541 if (!AdditionalValues.empty())
1542 break;
1543
1544 // New value and expr now represent this debuginfo.
1545 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue);
1546
1547 // Some kind of simplification occurred: check whether the operand of the
1548 // salvaged debug expression can be encoded in this DAG.
1549 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) {
1550 LLVM_DEBUG(
1551 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n"
1552 << *OrigV << "\nBy stripping back to:\n " << *V << "\n");
1553 return;
1554 }
1555 }
1556
1557 // This was the final opportunity to salvage this debug information, and it
1558 // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1559 // any earlier variable location.
1560 assert(OrigV && "V shouldn't be null");
1561 auto *Undef = UndefValue::get(OrigV->getType());
1562 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1563 DAG.AddDbgValue(SDV, false);
1564 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n "
1565 << printDDI(OrigV, DDI) << "\n");
1566}
1567
1569 DIExpression *Expr,
1570 DebugLoc DbgLoc,
1571 unsigned Order) {
1575 handleDebugValue(Poison, Var, NewExpr, DbgLoc, Order,
1576 /*IsVariadic*/ false);
1577}
1578
1580 DILocalVariable *Var,
1581 DIExpression *Expr, DebugLoc DbgLoc,
1582 unsigned Order, bool IsVariadic) {
1583 if (Values.empty())
1584 return true;
1585
1586 // Filter EntryValue locations out early.
1587 if (visitEntryValueDbgValue(Values, Var, Expr, DbgLoc))
1588 return true;
1589
1590 SmallVector<SDDbgOperand> LocationOps;
1591 SmallVector<SDNode *> Dependencies;
1592 for (const Value *V : Values) {
1593 // Constant value.
1594 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
1595 isa<ConstantPointerNull>(V)) {
1596 LocationOps.emplace_back(SDDbgOperand::fromConst(V));
1597 continue;
1598 }
1599
1600 // Look through IntToPtr constants.
1601 if (auto *CE = dyn_cast<ConstantExpr>(V))
1602 if (CE->getOpcode() == Instruction::IntToPtr) {
1603 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0)));
1604 continue;
1605 }
1606
1607 // If the Value is a frame index, we can create a FrameIndex debug value
1608 // without relying on the DAG at all.
1609 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1610 auto SI = FuncInfo.StaticAllocaMap.find(AI);
1611 if (SI != FuncInfo.StaticAllocaMap.end()) {
1612 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second));
1613 continue;
1614 }
1615 }
1616
1617 // Do not use getValue() in here; we don't want to generate code at
1618 // this point if it hasn't been done yet.
1619 SDValue N = NodeMap[V];
1620 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
1621 N = UnusedArgNodeMap[V];
1622
1623 if (N.getNode()) {
1624 // Only emit func arg dbg value for non-variadic dbg.values for now.
1625 if (!IsVariadic &&
1626 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc,
1627 FuncArgumentDbgValueKind::Value, N))
1628 return true;
1629 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
1630 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can
1631 // describe stack slot locations.
1632 //
1633 // Consider "int x = 0; int *px = &x;". There are two kinds of
1634 // interesting debug values here after optimization:
1635 //
1636 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
1637 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
1638 //
1639 // Both describe the direct values of their associated variables.
1640 Dependencies.push_back(N.getNode());
1641 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex()));
1642 continue;
1643 }
1644 LocationOps.emplace_back(
1645 SDDbgOperand::fromNode(N.getNode(), N.getResNo()));
1646 continue;
1647 }
1648
1650 // Special rules apply for the first dbg.values of parameter variables in a
1651 // function. Identify them by the fact they reference Argument Values, that
1652 // they're parameters, and they are parameters of the current function. We
1653 // need to let them dangle until they get an SDNode.
1654 bool IsParamOfFunc =
1655 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt();
1656 if (IsParamOfFunc)
1657 return false;
1658
1659 // The value is not used in this block yet (or it would have an SDNode).
1660 // We still want the value to appear for the user if possible -- if it has
1661 // an associated VReg, we can refer to that instead.
1662 auto VMI = FuncInfo.ValueMap.find(V);
1663 if (VMI != FuncInfo.ValueMap.end()) {
1664 unsigned Reg = VMI->second;
1665 // If this is a PHI node, it may be split up into several MI PHI nodes
1666 // (in FunctionLoweringInfo::set).
1667 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1668 V->getType(), std::nullopt);
1669 if (RFV.occupiesMultipleRegs()) {
1670 // FIXME: We could potentially support variadic dbg_values here.
1671 if (IsVariadic)
1672 return false;
1673 unsigned Offset = 0;
1674 unsigned BitsToDescribe = 0;
1675 if (auto VarSize = Var->getSizeInBits())
1676 BitsToDescribe = *VarSize;
1677 if (auto Fragment = Expr->getFragmentInfo())
1678 BitsToDescribe = Fragment->SizeInBits;
1679 for (const auto &RegAndSize : RFV.getRegsAndSizes()) {
1680 // Bail out if all bits are described already.
1681 if (Offset >= BitsToDescribe)
1682 break;
1683 // TODO: handle scalable vectors.
1684 unsigned RegisterSize = RegAndSize.second;
1685 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1686 ? BitsToDescribe - Offset
1687 : RegisterSize;
1688 auto FragmentExpr = DIExpression::createFragmentExpression(
1689 Expr, Offset, FragmentSize);
1690 if (!FragmentExpr)
1691 continue;
1693 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, Order);
1694 DAG.AddDbgValue(SDV, false);
1695 Offset += RegisterSize;
1696 }
1697 return true;
1698 }
1699 // We can use simple vreg locations for variadic dbg_values as well.
1700 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg));
1701 continue;
1702 }
1703 // We failed to create a SDDbgOperand for V.
1704 return false;
1705 }
1706
1707 // We have created a SDDbgOperand for each Value in Values.
1708 assert(!LocationOps.empty());
1709 SDDbgValue *SDV =
1710 DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies,
1711 /*IsIndirect=*/false, DbgLoc, Order, IsVariadic);
1712 DAG.AddDbgValue(SDV, /*isParameter=*/false);
1713 return true;
1714}
1715
1717 // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1718 for (auto &Pair : DanglingDebugInfoMap)
1719 for (auto &DDI : Pair.second)
1720 salvageUnresolvedDbgValue(const_cast<Value *>(Pair.first), DDI);
1722}
1723
1724/// getCopyFromRegs - If there was virtual register allocated for the value V
1725/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1728 SDValue Result;
1729
1730 if (It != FuncInfo.ValueMap.end()) {
1731 Register InReg = It->second;
1732
1734 DAG.getDataLayout(), InReg, Ty,
1735 std::nullopt); // This is not an ABI copy.
1736 SDValue Chain = DAG.getEntryNode();
1737 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1738 V);
1739 resolveDanglingDebugInfo(V, Result);
1740 }
1741
1742 return Result;
1743}
1744
1745/// getValue - Return an SDValue for the given Value.
1747 // If we already have an SDValue for this value, use it. It's important
1748 // to do this first, so that we don't create a CopyFromReg if we already
1749 // have a regular SDValue.
1750 SDValue &N = NodeMap[V];
1751 if (N.getNode()) return N;
1752
1753 // If there's a virtual register allocated and initialized for this
1754 // value, use it.
1755 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1756 return copyFromReg;
1757
1758 // Otherwise create a new SDValue and remember it.
1759 SDValue Val = getValueImpl(V);
1760 NodeMap[V] = Val;
1762 return Val;
1763}
1764
1765/// getNonRegisterValue - Return an SDValue for the given Value, but
1766/// don't look in FuncInfo.ValueMap for a virtual register.
1768 // If we already have an SDValue for this value, use it.
1769 SDValue &N = NodeMap[V];
1770 if (N.getNode()) {
1771 if (isIntOrFPConstant(N)) {
1772 // Remove the debug location from the node as the node is about to be used
1773 // in a location which may differ from the original debug location. This
1774 // is relevant to Constant and ConstantFP nodes because they can appear
1775 // as constant expressions inside PHI nodes.
1776 N->setDebugLoc(DebugLoc());
1777 }
1778 return N;
1779 }
1780
1781 // Otherwise create a new SDValue and remember it.
1782 SDValue Val = getValueImpl(V);
1783 NodeMap[V] = Val;
1785 return Val;
1786}
1787
1788/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1789/// Create an SDValue for the given value.
1792
1793 if (const Constant *C = dyn_cast<Constant>(V)) {
1794 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1795
1796 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1797 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1798
1799 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1800 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1801
1802 if (const ConstantPtrAuth *CPA = dyn_cast<ConstantPtrAuth>(C)) {
1804 getValue(CPA->getPointer()), getValue(CPA->getKey()),
1805 getValue(CPA->getAddrDiscriminator()),
1806 getValue(CPA->getDiscriminator()));
1807 }
1808
1809 if (isa<ConstantPointerNull>(C)) {
1810 unsigned AS = V->getType()->getPointerAddressSpace();
1811 return DAG.getConstant(0, getCurSDLoc(),
1812 TLI.getPointerTy(DAG.getDataLayout(), AS));
1813 }
1814
1815 if (match(C, m_VScale()))
1816 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
1817
1818 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1819 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1820
1821 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1822 return DAG.getUNDEF(VT);
1823
1824 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1825 visit(CE->getOpcode(), *CE);
1826 SDValue N1 = NodeMap[V];
1827 assert(N1.getNode() && "visit didn't populate the NodeMap!");
1828 return N1;
1829 }
1830
1831 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1832 SmallVector<SDValue, 4> Constants;
1833 for (const Use &U : C->operands()) {
1834 SDNode *Val = getValue(U).getNode();
1835 // If the operand is an empty aggregate, there are no values.
1836 if (!Val) continue;
1837 // Add each leaf value from the operand to the Constants list
1838 // to form a flattened list of all the values.
1839 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1840 Constants.push_back(SDValue(Val, i));
1841 }
1842
1843 return DAG.getMergeValues(Constants, getCurSDLoc());
1844 }
1845
1846 if (const ConstantDataSequential *CDS =
1847 dyn_cast<ConstantDataSequential>(C)) {
1849 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1850 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1851 // Add each leaf value from the operand to the Constants list
1852 // to form a flattened list of all the values.
1853 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1854 Ops.push_back(SDValue(Val, i));
1855 }
1856
1857 if (isa<ArrayType>(CDS->getType()))
1858 return DAG.getMergeValues(Ops, getCurSDLoc());
1859 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1860 }
1861
1862 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1863 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1864 "Unknown struct or array constant!");
1865
1866 SmallVector<EVT, 4> ValueVTs;
1867 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1868 unsigned NumElts = ValueVTs.size();
1869 if (NumElts == 0)
1870 return SDValue(); // empty struct
1871 SmallVector<SDValue, 4> Constants(NumElts);
1872 for (unsigned i = 0; i != NumElts; ++i) {
1873 EVT EltVT = ValueVTs[i];
1874 if (isa<UndefValue>(C))
1875 Constants[i] = DAG.getUNDEF(EltVT);
1876 else if (EltVT.isFloatingPoint())
1877 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1878 else
1879 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1880 }
1881
1882 return DAG.getMergeValues(Constants, getCurSDLoc());
1883 }
1884
1885 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1886 return DAG.getBlockAddress(BA, VT);
1887
1888 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C))
1889 return getValue(Equiv->getGlobalValue());
1890
1891 if (const auto *NC = dyn_cast<NoCFIValue>(C))
1892 return getValue(NC->getGlobalValue());
1893
1894 if (VT == MVT::aarch64svcount) {
1895 assert(C->isNullValue() && "Can only zero this target type!");
1896 return DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT,
1897 DAG.getConstant(0, getCurSDLoc(), MVT::nxv16i1));
1898 }
1899
1900 if (VT.isRISCVVectorTuple()) {
1901 assert(C->isNullValue() && "Can only zero this target type!");
1902 return NodeMap[V] = DAG.getNode(
1904 DAG.getNode(
1906 EVT::getVectorVT(*DAG.getContext(), MVT::i8,
1908 true),
1910 }
1911
1912 VectorType *VecTy = cast<VectorType>(V->getType());
1913
1914 // Now that we know the number and type of the elements, get that number of
1915 // elements into the Ops array based on what kind of constant it is.
1916 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1918 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1919 for (unsigned i = 0; i != NumElements; ++i)
1920 Ops.push_back(getValue(CV->getOperand(i)));
1921
1922 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1923 }
1924
1925 if (isa<ConstantAggregateZero>(C)) {
1926 EVT EltVT =
1928
1929 SDValue Op;
1930 if (EltVT.isFloatingPoint())
1931 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1932 else
1933 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1934
1935 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op);
1936 }
1937
1938 llvm_unreachable("Unknown vector constant");
1939 }
1940
1941 // If this is a static alloca, generate it as the frameindex instead of
1942 // computation.
1943 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1945 FuncInfo.StaticAllocaMap.find(AI);
1946 if (SI != FuncInfo.StaticAllocaMap.end())
1947 return DAG.getFrameIndex(
1948 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType()));
1949 }
1950
1951 // If this is an instruction which fast-isel has deferred, select it now.
1952 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1954
1955 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1956 Inst->getType(), std::nullopt);
1957 SDValue Chain = DAG.getEntryNode();
1958 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1959 }
1960
1961 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V))
1962 return DAG.getMDNode(cast<MDNode>(MD->getMetadata()));
1963
1964 if (const auto *BB = dyn_cast<BasicBlock>(V))
1965 return DAG.getBasicBlock(FuncInfo.getMBB(BB));
1966
1967 llvm_unreachable("Can't get register for value!");
1968}
1969
1970void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1972 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1973 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1974 bool IsSEH = isAsynchronousEHPersonality(Pers);
1975 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1976 if (!IsSEH)
1977 CatchPadMBB->setIsEHScopeEntry();
1978 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1979 if (IsMSVCCXX || IsCoreCLR)
1980 CatchPadMBB->setIsEHFuncletEntry();
1981}
1982
1983void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1984 // Update machine-CFG edge.
1985 MachineBasicBlock *TargetMBB = FuncInfo.getMBB(I.getSuccessor());
1986 FuncInfo.MBB->addSuccessor(TargetMBB);
1987 TargetMBB->setIsEHCatchretTarget(true);
1989
1991 bool IsSEH = isAsynchronousEHPersonality(Pers);
1992 if (IsSEH) {
1993 // If this is not a fall-through branch or optimizations are switched off,
1994 // emit the branch.
1995 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1997 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1998 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1999 return;
2000 }
2001
2002 // Figure out the funclet membership for the catchret's successor.
2003 // This will be used by the FuncletLayout pass to determine how to order the
2004 // BB's.
2005 // A 'catchret' returns to the outer scope's color.
2006 Value *ParentPad = I.getCatchSwitchParentPad();
2007 const BasicBlock *SuccessorColor;
2008 if (isa<ConstantTokenNone>(ParentPad))
2009 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
2010 else
2011 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
2012 assert(SuccessorColor && "No parent funclet for catchret!");
2013 MachineBasicBlock *SuccessorColorMBB = FuncInfo.getMBB(SuccessorColor);
2014 assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
2015
2016 // Create the terminator node.
2018 getControlRoot(), DAG.getBasicBlock(TargetMBB),
2019 DAG.getBasicBlock(SuccessorColorMBB));
2020 DAG.setRoot(Ret);
2021}
2022
2023void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
2024 // Don't emit any special code for the cleanuppad instruction. It just marks
2025 // the start of an EH scope/funclet.
2028 if (Pers != EHPersonality::Wasm_CXX) {
2031 }
2032}
2033
2034// In wasm EH, even though a catchpad may not catch an exception if a tag does
2035// not match, it is OK to add only the first unwind destination catchpad to the
2036// successors, because there will be at least one invoke instruction within the
2037// catch scope that points to the next unwind destination, if one exists, so
2038// CFGSort cannot mess up with BB sorting order.
2039// (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic
2040// call within them, and catchpads only consisting of 'catch (...)' have a
2041// '__cxa_end_catch' call within them, both of which generate invokes in case
2042// the next unwind destination exists, i.e., the next unwind destination is not
2043// the caller.)
2044//
2045// Having at most one EH pad successor is also simpler and helps later
2046// transformations.
2047//
2048// For example,
2049// current:
2050// invoke void @foo to ... unwind label %catch.dispatch
2051// catch.dispatch:
2052// %0 = catchswitch within ... [label %catch.start] unwind label %next
2053// catch.start:
2054// ...
2055// ... in this BB or some other child BB dominated by this BB there will be an
2056// invoke that points to 'next' BB as an unwind destination
2057//
2058// next: ; We don't need to add this to 'current' BB's successor
2059// ...
2061 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2062 BranchProbability Prob,
2063 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2064 &UnwindDests) {
2065 while (EHPadBB) {
2066 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2067 if (isa<CleanupPadInst>(Pad)) {
2068 // Stop on cleanup pads.
2069 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2070 UnwindDests.back().first->setIsEHScopeEntry();
2071 break;
2072 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2073 // Add the catchpad handlers to the possible destinations. We don't
2074 // continue to the unwind destination of the catchswitch for wasm.
2075 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2076 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2077 UnwindDests.back().first->setIsEHScopeEntry();
2078 }
2079 break;
2080 } else {
2081 continue;
2082 }
2083 }
2084}
2085
2086/// When an invoke or a cleanupret unwinds to the next EH pad, there are
2087/// many places it could ultimately go. In the IR, we have a single unwind
2088/// destination, but in the machine CFG, we enumerate all the possible blocks.
2089/// This function skips over imaginary basic blocks that hold catchswitch
2090/// instructions, and finds all the "real" machine
2091/// basic block destinations. As those destinations may not be successors of
2092/// EHPadBB, here we also calculate the edge probability to those destinations.
2093/// The passed-in Prob is the edge probability to EHPadBB.
2095 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
2096 BranchProbability Prob,
2097 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
2098 &UnwindDests) {
2099 EHPersonality Personality =
2101 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
2102 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
2103 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
2104 bool IsSEH = isAsynchronousEHPersonality(Personality);
2105
2106 if (IsWasmCXX) {
2107 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
2108 assert(UnwindDests.size() <= 1 &&
2109 "There should be at most one unwind destination for wasm");
2110 return;
2111 }
2112
2113 while (EHPadBB) {
2114 const Instruction *Pad = EHPadBB->getFirstNonPHI();
2115 BasicBlock *NewEHPadBB = nullptr;
2116 if (isa<LandingPadInst>(Pad)) {
2117 // Stop on landingpads. They are not funclets.
2118 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2119 break;
2120 } else if (isa<CleanupPadInst>(Pad)) {
2121 // Stop on cleanup pads. Cleanups are always funclet entries for all known
2122 // personalities.
2123 UnwindDests.emplace_back(FuncInfo.getMBB(EHPadBB), Prob);
2124 UnwindDests.back().first->setIsEHScopeEntry();
2125 UnwindDests.back().first->setIsEHFuncletEntry();
2126 break;
2127 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
2128 // Add the catchpad handlers to the possible destinations.
2129 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
2130 UnwindDests.emplace_back(FuncInfo.getMBB(CatchPadBB), Prob);
2131 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
2132 if (IsMSVCCXX || IsCoreCLR)
2133 UnwindDests.back().first->setIsEHFuncletEntry();
2134 if (!IsSEH)
2135 UnwindDests.back().first->setIsEHScopeEntry();
2136 }
2137 NewEHPadBB = CatchSwitch->getUnwindDest();
2138 } else {
2139 continue;
2140 }
2141
2142 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2143 if (BPI && NewEHPadBB)
2144 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
2145 EHPadBB = NewEHPadBB;
2146 }
2147}
2148
2149void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
2150 // Update successor info.
2152 auto UnwindDest = I.getUnwindDest();
2154 BranchProbability UnwindDestProb =
2155 (BPI && UnwindDest)
2156 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
2158 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
2159 for (auto &UnwindDest : UnwindDests) {
2160 UnwindDest.first->setIsEHPad();
2161 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
2162 }
2164
2165 // Create the terminator node.
2166 MachineBasicBlock *CleanupPadMBB =
2167 FuncInfo.getMBB(I.getCleanupPad()->getParent());
2169 getControlRoot(), DAG.getBasicBlock(CleanupPadMBB));
2170 DAG.setRoot(Ret);
2171}
2172
2173void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
2174 report_fatal_error("visitCatchSwitch not yet implemented!");
2175}
2176
2177void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
2179 auto &DL = DAG.getDataLayout();
2180 SDValue Chain = getControlRoot();
2183
2184 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
2185 // lower
2186 //
2187 // %val = call <ty> @llvm.experimental.deoptimize()
2188 // ret <ty> %val
2189 //
2190 // differently.
2191 if (I.getParent()->getTerminatingDeoptimizeCall()) {
2193 return;
2194 }
2195
2196 if (!FuncInfo.CanLowerReturn) {
2197 Register DemoteReg = FuncInfo.DemoteRegister;
2198
2199 // Emit a store of the return value through the virtual register.
2200 // Leave Outs empty so that LowerReturn won't try to load return
2201 // registers the usual way.
2202 MVT PtrValueVT = TLI.getPointerTy(DL, DL.getAllocaAddrSpace());
2203 SDValue RetPtr =
2204 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVT);
2205 SDValue RetOp = getValue(I.getOperand(0));
2206
2207 SmallVector<EVT, 4> ValueVTs, MemVTs;
2209 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
2210 &Offsets, 0);
2211 unsigned NumValues = ValueVTs.size();
2212
2213 SmallVector<SDValue, 4> Chains(NumValues);
2214 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType());
2215 for (unsigned i = 0; i != NumValues; ++i) {
2216 // An aggregate return value cannot wrap around the address space, so
2217 // offsets to its parts don't wrap either.
2219 TypeSize::getFixed(Offsets[i]));
2220
2221 SDValue Val = RetOp.getValue(RetOp.getResNo() + i);
2222 if (MemVTs[i] != ValueVTs[i])
2223 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
2224 Chains[i] = DAG.getStore(
2225 Chain, getCurSDLoc(), Val,
2226 // FIXME: better loc info would be nice.
2228 commonAlignment(BaseAlign, Offsets[i]));
2229 }
2230
2232 MVT::Other, Chains);
2233 } else if (I.getNumOperands() != 0) {
2234 SmallVector<EVT, 4> ValueVTs;
2235 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
2236 unsigned NumValues = ValueVTs.size();
2237 if (NumValues) {
2238 SDValue RetOp = getValue(I.getOperand(0));
2239
2240 const Function *F = I.getParent()->getParent();
2241
2242 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
2243 I.getOperand(0)->getType(), F->getCallingConv(),
2244 /*IsVarArg*/ false, DL);
2245
2246 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
2247 if (F->getAttributes().hasRetAttr(Attribute::SExt))
2248 ExtendKind = ISD::SIGN_EXTEND;
2249 else if (F->getAttributes().hasRetAttr(Attribute::ZExt))
2250 ExtendKind = ISD::ZERO_EXTEND;
2251
2252 LLVMContext &Context = F->getContext();
2253 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
2254
2255 for (unsigned j = 0; j != NumValues; ++j) {
2256 EVT VT = ValueVTs[j];
2257
2258 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
2259 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
2260
2261 CallingConv::ID CC = F->getCallingConv();
2262
2263 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
2264 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
2265 SmallVector<SDValue, 4> Parts(NumParts);
2267 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
2268 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
2269
2270 // 'inreg' on function refers to return value
2272 if (RetInReg)
2273 Flags.setInReg();
2274
2275 if (I.getOperand(0)->getType()->isPointerTy()) {
2276 Flags.setPointer();
2277 Flags.setPointerAddrSpace(
2278 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
2279 }
2280
2281 if (NeedsRegBlock) {
2282 Flags.setInConsecutiveRegs();
2283 if (j == NumValues - 1)
2284 Flags.setInConsecutiveRegsLast();
2285 }
2286
2287 // Propagate extension type if any
2288 if (ExtendKind == ISD::SIGN_EXTEND)
2289 Flags.setSExt();
2290 else if (ExtendKind == ISD::ZERO_EXTEND)
2291 Flags.setZExt();
2292 else if (F->getAttributes().hasRetAttr(Attribute::NoExt))
2293 Flags.setNoExt();
2294
2295 for (unsigned i = 0; i < NumParts; ++i) {
2296 Outs.push_back(ISD::OutputArg(Flags,
2297 Parts[i].getValueType().getSimpleVT(),
2298 VT, /*isfixed=*/true, 0, 0));
2299 OutVals.push_back(Parts[i]);
2300 }
2301 }
2302 }
2303 }
2304
2305 // Push in swifterror virtual register as the last element of Outs. This makes
2306 // sure swifterror virtual register will be returned in the swifterror
2307 // physical register.
2308 const Function *F = I.getParent()->getParent();
2309 if (TLI.supportSwiftError() &&
2310 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
2311 assert(SwiftError.getFunctionArg() && "Need a swift error argument");
2313 Flags.setSwiftError();
2315 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
2316 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
2317 // Create SDNode for the swifterror virtual register.
2318 OutVals.push_back(
2321 EVT(TLI.getPointerTy(DL))));
2322 }
2323
2324 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
2325 CallingConv::ID CallConv =
2328 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
2329
2330 // Verify that the target's LowerReturn behaved as expected.
2331 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
2332 "LowerReturn didn't return a valid chain!");
2333
2334 // Update the DAG with the new chain value resulting from return lowering.
2335 DAG.setRoot(Chain);
2336}
2337
2338/// CopyToExportRegsIfNeeded - If the given value has virtual registers
2339/// created for it, emit nodes to copy the value into the virtual
2340/// registers.
2342 // Skip empty types
2343 if (V->getType()->isEmptyTy())
2344 return;
2345
2347 if (VMI != FuncInfo.ValueMap.end()) {
2348 assert((!V->use_empty() || isa<CallBrInst>(V)) &&
2349 "Unused value assigned virtual registers!");
2350 CopyValueToVirtualRegister(V, VMI->second);
2351 }
2352}
2353
2354/// ExportFromCurrentBlock - If this condition isn't known to be exported from
2355/// the current basic block, add it to ValueMap now so that we'll get a
2356/// CopyTo/FromReg.
2358 // No need to export constants.
2359 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
2360
2361 // Already exported?
2362 if (FuncInfo.isExportedInst(V)) return;
2363
2366}
2367
2369 const BasicBlock *FromBB) {
2370 // The operands of the setcc have to be in this block. We don't know
2371 // how to export them from some other block.
2372 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
2373 // Can export from current BB.
2374 if (VI->getParent() == FromBB)
2375 return true;
2376
2377 // Is already exported, noop.
2378 return FuncInfo.isExportedInst(V);
2379 }
2380
2381 // If this is an argument, we can export it if the BB is the entry block or
2382 // if it is already exported.
2383 if (isa<Argument>(V)) {
2384 if (FromBB->isEntryBlock())
2385 return true;
2386
2387 // Otherwise, can only export this if it is already exported.
2388 return FuncInfo.isExportedInst(V);
2389 }
2390
2391 // Otherwise, constants can always be exported.
2392 return true;
2393}
2394
2395/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
2397SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
2398 const MachineBasicBlock *Dst) const {
2400 const BasicBlock *SrcBB = Src->getBasicBlock();
2401 const BasicBlock *DstBB = Dst->getBasicBlock();
2402 if (!BPI) {
2403 // If BPI is not available, set the default probability as 1 / N, where N is
2404 // the number of successors.
2405 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
2406 return BranchProbability(1, SuccSize);
2407 }
2408 return BPI->getEdgeProbability(SrcBB, DstBB);
2409}
2410
2411void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2412 MachineBasicBlock *Dst,
2413 BranchProbability Prob) {
2414 if (!FuncInfo.BPI)
2415 Src->addSuccessorWithoutProb(Dst);
2416 else {
2417 if (Prob.isUnknown())
2418 Prob = getEdgeProbability(Src, Dst);
2419 Src->addSuccessor(Dst, Prob);
2420 }
2421}
2422
2423static bool InBlock(const Value *V, const BasicBlock *BB) {
2424 if (const Instruction *I = dyn_cast<Instruction>(V))
2425 return I->getParent() == BB;
2426 return true;
2427}
2428
2429/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2430/// This function emits a branch and is used at the leaves of an OR or an
2431/// AND operator tree.
2432void
2435 MachineBasicBlock *FBB,
2436 MachineBasicBlock *CurBB,
2437 MachineBasicBlock *SwitchBB,
2438 BranchProbability TProb,
2439 BranchProbability FProb,
2440 bool InvertCond) {
2441 const BasicBlock *BB = CurBB->getBasicBlock();
2442
2443 // If the leaf of the tree is a comparison, merge the condition into
2444 // the caseblock.
2445 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2446 // The operands of the cmp have to be in this block. We don't know
2447 // how to export them from some other block. If this is the first block
2448 // of the sequence, no exporting is needed.
2449 if (CurBB == SwitchBB ||
2450 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
2451 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
2452 ISD::CondCode Condition;
2453 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2454 ICmpInst::Predicate Pred =
2455 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
2456 Condition = getICmpCondCode(Pred);
2457 } else {
2458 const FCmpInst *FC = cast<FCmpInst>(Cond);
2459 FCmpInst::Predicate Pred =
2460 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
2461 Condition = getFCmpCondCode(Pred);
2462 if (TM.Options.NoNaNsFPMath)
2463 Condition = getFCmpCodeWithoutNaN(Condition);
2464 }
2465
2466 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2467 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2468 SL->SwitchCases.push_back(CB);
2469 return;
2470 }
2471 }
2472
2473 // Create a CaseBlock record representing this branch.
2474 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
2476 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2477 SL->SwitchCases.push_back(CB);
2478}
2479
2480// Collect dependencies on V recursively. This is used for the cost analysis in
2481// `shouldKeepJumpConditionsTogether`.
2485 unsigned Depth = 0) {
2486 // Return false if we have an incomplete count.
2488 return false;
2489
2490 auto *I = dyn_cast<Instruction>(V);
2491 if (I == nullptr)
2492 return true;
2493
2494 if (Necessary != nullptr) {
2495 // This instruction is necessary for the other side of the condition so
2496 // don't count it.
2497 if (Necessary->contains(I))
2498 return true;
2499 }
2500
2501 // Already added this dep.
2502 if (!Deps->try_emplace(I, false).second)
2503 return true;
2504
2505 for (unsigned OpIdx = 0, E = I->getNumOperands(); OpIdx < E; ++OpIdx)
2506 if (!collectInstructionDeps(Deps, I->getOperand(OpIdx), Necessary,
2507 Depth + 1))
2508 return false;
2509 return true;
2510}
2511
2513 const FunctionLoweringInfo &FuncInfo, const BranchInst &I,
2514 Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs,
2516 if (I.getNumSuccessors() != 2)
2517 return false;
2518
2519 if (!I.isConditional())
2520 return false;
2521
2522 if (Params.BaseCost < 0)
2523 return false;
2524
2525 // Baseline cost.
2526 InstructionCost CostThresh = Params.BaseCost;
2527
2528 BranchProbabilityInfo *BPI = nullptr;
2529 if (Params.LikelyBias || Params.UnlikelyBias)
2530 BPI = FuncInfo.BPI;
2531 if (BPI != nullptr) {
2532 // See if we are either likely to get an early out or compute both lhs/rhs
2533 // of the condition.
2534 BasicBlock *IfFalse = I.getSuccessor(0);
2535 BasicBlock *IfTrue = I.getSuccessor(1);
2536
2537 std::optional<bool> Likely;
2538 if (BPI->isEdgeHot(I.getParent(), IfTrue))
2539 Likely = true;
2540 else if (BPI->isEdgeHot(I.getParent(), IfFalse))
2541 Likely = false;
2542
2543 if (Likely) {
2544 if (Opc == (*Likely ? Instruction::And : Instruction::Or))
2545 // Its likely we will have to compute both lhs and rhs of condition
2546 CostThresh += Params.LikelyBias;
2547 else {
2548 if (Params.UnlikelyBias < 0)
2549 return false;
2550 // Its likely we will get an early out.
2551 CostThresh -= Params.UnlikelyBias;
2552 }
2553 }
2554 }
2555
2556 if (CostThresh <= 0)
2557 return false;
2558
2559 // Collect "all" instructions that lhs condition is dependent on.
2560 // Use map for stable iteration (to avoid non-determanism of iteration of
2561 // SmallPtrSet). The `bool` value is just a dummy.
2563 collectInstructionDeps(&LhsDeps, Lhs);
2564 // Collect "all" instructions that rhs condition is dependent on AND are
2565 // dependencies of lhs. This gives us an estimate on which instructions we
2566 // stand to save by splitting the condition.
2567 if (!collectInstructionDeps(&RhsDeps, Rhs, &LhsDeps))
2568 return false;
2569 // Add the compare instruction itself unless its a dependency on the LHS.
2570 if (const auto *RhsI = dyn_cast<Instruction>(Rhs))
2571 if (!LhsDeps.contains(RhsI))
2572 RhsDeps.try_emplace(RhsI, false);
2573
2574 const auto &TLI = DAG.getTargetLoweringInfo();
2575 const auto &TTI =
2576 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
2577
2578 InstructionCost CostOfIncluding = 0;
2579 // See if this instruction will need to computed independently of whether RHS
2580 // is.
2581 Value *BrCond = I.getCondition();
2582 auto ShouldCountInsn = [&RhsDeps, &BrCond](const Instruction *Ins) {
2583 for (const auto *U : Ins->users()) {
2584 // If user is independent of RHS calculation we don't need to count it.
2585 if (auto *UIns = dyn_cast<Instruction>(U))
2586 if (UIns != BrCond && !RhsDeps.contains(UIns))
2587 return false;
2588 }
2589 return true;
2590 };
2591
2592 // Prune instructions from RHS Deps that are dependencies of unrelated
2593 // instructions. The value (SelectionDAG::MaxRecursionDepth) is fairly
2594 // arbitrary and just meant to cap the how much time we spend in the pruning
2595 // loop. Its highly unlikely to come into affect.
2596 const unsigned MaxPruneIters = SelectionDAG::MaxRecursionDepth;
2597 // Stop after a certain point. No incorrectness from including too many
2598 // instructions.
2599 for (unsigned PruneIters = 0; PruneIters < MaxPruneIters; ++PruneIters) {
2600 const Instruction *ToDrop = nullptr;
2601 for (const auto &InsPair : RhsDeps) {
2602 if (!ShouldCountInsn(InsPair.first)) {
2603 ToDrop = InsPair.first;
2604 break;
2605 }
2606 }
2607 if (ToDrop == nullptr)
2608 break;
2609 RhsDeps.erase(ToDrop);
2610 }
2611
2612 for (const auto &InsPair : RhsDeps) {
2613 // Finally accumulate latency that we can only attribute to computing the
2614 // RHS condition. Use latency because we are essentially trying to calculate
2615 // the cost of the dependency chain.
2616 // Possible TODO: We could try to estimate ILP and make this more precise.
2617 CostOfIncluding +=
2619
2620 if (CostOfIncluding > CostThresh)
2621 return false;
2622 }
2623 return true;
2624}
2625
2628 MachineBasicBlock *FBB,
2629 MachineBasicBlock *CurBB,
2630 MachineBasicBlock *SwitchBB,
2632 BranchProbability TProb,
2633 BranchProbability FProb,
2634 bool InvertCond) {
2635 // Skip over not part of the tree and remember to invert op and operands at
2636 // next level.
2637 Value *NotCond;
2638 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2639 InBlock(NotCond, CurBB->getBasicBlock())) {
2640 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2641 !InvertCond);
2642 return;
2643 }
2644
2645 const Instruction *BOp = dyn_cast<Instruction>(Cond);
2646 const Value *BOpOp0, *BOpOp1;
2647 // Compute the effective opcode for Cond, taking into account whether it needs
2648 // to be inverted, e.g.
2649 // and (not (or A, B)), C
2650 // gets lowered as
2651 // and (and (not A, not B), C)
2653 if (BOp) {
2654 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))
2655 ? Instruction::And
2656 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))
2657 ? Instruction::Or
2659 if (InvertCond) {
2660 if (BOpc == Instruction::And)
2661 BOpc = Instruction::Or;
2662 else if (BOpc == Instruction::Or)
2663 BOpc = Instruction::And;
2664 }
2665 }
2666
2667 // If this node is not part of the or/and tree, emit it as a branch.
2668 // Note that all nodes in the tree should have same opcode.
2669 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();
2670 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||
2671 !InBlock(BOpOp0, CurBB->getBasicBlock()) ||
2672 !InBlock(BOpOp1, CurBB->getBasicBlock())) {
2673 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2674 TProb, FProb, InvertCond);
2675 return;
2676 }
2677
2678 // Create TmpBB after CurBB.
2679 MachineFunction::iterator BBI(CurBB);
2682 CurBB->getParent()->insert(++BBI, TmpBB);
2683
2684 if (Opc == Instruction::Or) {
2685 // Codegen X | Y as:
2686 // BB1:
2687 // jmp_if_X TBB
2688 // jmp TmpBB
2689 // TmpBB:
2690 // jmp_if_Y TBB
2691 // jmp FBB
2692 //
2693
2694 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2695 // The requirement is that
2696 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2697 // = TrueProb for original BB.
2698 // Assuming the original probabilities are A and B, one choice is to set
2699 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2700 // A/(1+B) and 2B/(1+B). This choice assumes that
2701 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2702 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2703 // TmpBB, but the math is more complicated.
2704
2705 auto NewTrueProb = TProb / 2;
2706 auto NewFalseProb = TProb / 2 + FProb;
2707 // Emit the LHS condition.
2708 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,
2709 NewFalseProb, InvertCond);
2710
2711 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2712 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2713 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2714 // Emit the RHS condition into TmpBB.
2715 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2716 Probs[1], InvertCond);
2717 } else {
2718 assert(Opc == Instruction::And && "Unknown merge op!");
2719 // Codegen X & Y as:
2720 // BB1:
2721 // jmp_if_X TmpBB
2722 // jmp FBB
2723 // TmpBB:
2724 // jmp_if_Y TBB
2725 // jmp FBB
2726 //
2727 // This requires creation of TmpBB after CurBB.
2728
2729 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2730 // The requirement is that
2731 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2732 // = FalseProb for original BB.
2733 // Assuming the original probabilities are A and B, one choice is to set
2734 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2735 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2736 // TrueProb for BB1 * FalseProb for TmpBB.
2737
2738 auto NewTrueProb = TProb + FProb / 2;
2739 auto NewFalseProb = FProb / 2;
2740 // Emit the LHS condition.
2741 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,
2742 NewFalseProb, InvertCond);
2743
2744 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2745 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2746 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2747 // Emit the RHS condition into TmpBB.
2748 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],
2749 Probs[1], InvertCond);
2750 }
2751}
2752
2753/// If the set of cases should be emitted as a series of branches, return true.
2754/// If we should emit this as a bunch of and/or'd together conditions, return
2755/// false.
2756bool
2757SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2758 if (Cases.size() != 2) return true;
2759
2760 // If this is two comparisons of the same values or'd or and'd together, they
2761 // will get folded into a single comparison, so don't emit two blocks.
2762 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2763 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
2764 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
2765 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
2766 return false;
2767 }
2768
2769 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2770 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2771 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2772 Cases[0].CC == Cases[1].CC &&
2773 isa<Constant>(Cases[0].CmpRHS) &&
2774 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
2775 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
2776 return false;
2777 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
2778 return false;
2779 }
2780
2781 return true;
2782}
2783
2784void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2786
2787 // Update machine-CFG edges.
2788 MachineBasicBlock *Succ0MBB = FuncInfo.getMBB(I.getSuccessor(0));
2789
2790 if (I.isUnconditional()) {
2791 // Update machine-CFG edges.
2792 BrMBB->addSuccessor(Succ0MBB);
2793
2794 // If this is not a fall-through branch or optimizations are switched off,
2795 // emit the branch.
2796 if (Succ0MBB != NextBlock(BrMBB) ||
2798 auto Br = DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
2799 getControlRoot(), DAG.getBasicBlock(Succ0MBB));
2800 setValue(&I, Br);
2801 DAG.setRoot(Br);
2802 }
2803
2804 return;
2805 }
2806
2807 // If this condition is one of the special cases we handle, do special stuff
2808 // now.
2809 const Value *CondVal = I.getCondition();
2810 MachineBasicBlock *Succ1MBB = FuncInfo.getMBB(I.getSuccessor(1));
2811
2812 // If this is a series of conditions that are or'd or and'd together, emit
2813 // this as a sequence of branches instead of setcc's with and/or operations.
2814 // As long as jumps are not expensive (exceptions for multi-use logic ops,
2815 // unpredictable branches, and vector extracts because those jumps are likely
2816 // expensive for any target), this should improve performance.
2817 // For example, instead of something like:
2818 // cmp A, B
2819 // C = seteq
2820 // cmp D, E
2821 // F = setle
2822 // or C, F
2823 // jnz foo
2824 // Emit:
2825 // cmp A, B
2826 // je foo
2827 // cmp D, E
2828 // jle foo
2829 bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
2830 const Instruction *BOp = dyn_cast<Instruction>(CondVal);
2831 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2832 BOp->hasOneUse() && !IsUnpredictable) {
2833 Value *Vec;
2834 const Value *BOp0, *BOp1;
2836 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))
2837 Opcode = Instruction::And;
2838 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))
2839 Opcode = Instruction::Or;
2840
2841 if (Opcode &&
2842 !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&
2843 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value()))) &&
2845 FuncInfo, I, Opcode, BOp0, BOp1,
2847 Opcode, BOp0, BOp1))) {
2848 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode,
2849 getEdgeProbability(BrMBB, Succ0MBB),
2850 getEdgeProbability(BrMBB, Succ1MBB),
2851 /*InvertCond=*/false);
2852 // If the compares in later blocks need to use values not currently
2853 // exported from this block, export them now. This block should always
2854 // be the first entry.
2855 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2856
2857 // Allow some cases to be rejected.
2858 if (ShouldEmitAsBranches(SL->SwitchCases)) {
2859 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) {
2860 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2861 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2862 }
2863
2864 // Emit the branch for this block.
2865 visitSwitchCase(SL->SwitchCases[0], BrMBB);
2866 SL->SwitchCases.erase(SL->SwitchCases.begin());
2867 return;
2868 }
2869
2870 // Okay, we decided not to do this, remove any inserted MBB's and clear
2871 // SwitchCases.
2872 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i)
2873 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2874
2875 SL->SwitchCases.clear();
2876 }
2877 }
2878
2879 // Create a CaseBlock record representing this branch.
2881 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2883 IsUnpredictable);
2884
2885 // Use visitSwitchCase to actually insert the fast branch sequence for this
2886 // cond branch.
2887 visitSwitchCase(CB, BrMBB);
2888}
2889
2890/// visitSwitchCase - Emits the necessary code to represent a single node in
2891/// the binary search tree resulting from lowering a switch instruction.
2893 MachineBasicBlock *SwitchBB) {
2894 SDValue Cond;
2895 SDValue CondLHS = getValue(CB.CmpLHS);
2896 SDLoc dl = CB.DL;
2897
2898 if (CB.CC == ISD::SETTRUE) {
2899 // Branch or fall through to TrueBB.
2900 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2901 SwitchBB->normalizeSuccProbs();
2902 if (CB.TrueBB != NextBlock(SwitchBB)) {
2903 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2904 DAG.getBasicBlock(CB.TrueBB)));
2905 }
2906 return;
2907 }
2908
2909 auto &TLI = DAG.getTargetLoweringInfo();
2910 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2911
2912 // Build the setcc now.
2913 if (!CB.CmpMHS) {
2914 // Fold "(X == true)" to X and "(X == false)" to !X to
2915 // handle common cases produced by branch lowering.
2916 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2917 CB.CC == ISD::SETEQ)
2918 Cond = CondLHS;
2919 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2920 CB.CC == ISD::SETEQ) {
2921 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2922 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2923 } else {
2924 SDValue CondRHS = getValue(CB.CmpRHS);
2925
2926 // If a pointer's DAG type is larger than its memory type then the DAG
2927 // values are zero-extended. This breaks signed comparisons so truncate
2928 // back to the underlying type before doing the compare.
2929 if (CondLHS.getValueType() != MemVT) {
2930 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2931 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2932 }
2933 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2934 }
2935 } else {
2936 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2937
2938 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2939 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2940
2941 SDValue CmpOp = getValue(CB.CmpMHS);
2942 EVT VT = CmpOp.getValueType();
2943
2944 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2945 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2946 ISD::SETLE);
2947 } else {
2948 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2949 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2950 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2951 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2952 }
2953 }
2954
2955 // Update successor info
2956 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2957 // TrueBB and FalseBB are always different unless the incoming IR is
2958 // degenerate. This only happens when running llc on weird IR.
2959 if (CB.TrueBB != CB.FalseBB)
2960 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2961 SwitchBB->normalizeSuccProbs();
2962
2963 // If the lhs block is the next block, invert the condition so that we can
2964 // fall through to the lhs instead of the rhs block.
2965 if (CB.TrueBB == NextBlock(SwitchBB)) {
2966 std::swap(CB.TrueBB, CB.FalseBB);
2967 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2968 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2969 }
2970
2971 SDNodeFlags Flags;
2972 Flags.setUnpredictable(CB.IsUnpredictable);
2973 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
2974 Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
2975
2976 setValue(CurInst, BrCond);
2977
2978 // Insert the false branch. Do this even if it's a fall through branch,
2979 // this makes it easier to do DAG optimizations which require inverting
2980 // the branch condition.
2981 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2983
2984 DAG.setRoot(BrCond);
2985}
2986
2987/// visitJumpTable - Emit JumpTable node in the current MBB
2989 // Emit the code for the jump table
2990 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
2991 assert(JT.Reg && "Should lower JT Header first!");
2993 SDValue Index = DAG.getCopyFromReg(getControlRoot(), *JT.SL, JT.Reg, PTy);
2994 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2995 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, *JT.SL, MVT::Other,
2996 Index.getValue(1), Table, Index);
2997 DAG.setRoot(BrJumpTable);
2998}
2999
3000/// visitJumpTableHeader - This function emits necessary code to produce index
3001/// in the JumpTable from switch case.
3003 JumpTableHeader &JTH,
3004 MachineBasicBlock *SwitchBB) {
3005 assert(JT.SL && "Should set SDLoc for SelectionDAG!");
3006 const SDLoc &dl = *JT.SL;
3007
3008 // Subtract the lowest switch case value from the value being switched on.
3009 SDValue SwitchOp = getValue(JTH.SValue);
3010 EVT VT = SwitchOp.getValueType();
3011 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
3012 DAG.getConstant(JTH.First, dl, VT));
3013
3014 // The SDNode we just created, which holds the value being switched on minus
3015 // the smallest case value, needs to be copied to a virtual register so it
3016 // can be used as an index into the jump table in a subsequent basic block.
3017 // This value may be smaller or larger than the target's pointer type, and
3018 // therefore require extension or truncating.
3020 SwitchOp =
3022
3023 Register JumpTableReg =
3025 SDValue CopyTo =
3026 DAG.getCopyToReg(getControlRoot(), dl, JumpTableReg, SwitchOp);
3027 JT.Reg = JumpTableReg;
3028
3029 if (!JTH.FallthroughUnreachable) {
3030 // Emit the range check for the jump table, and branch to the default block
3031 // for the switch statement if the value being switched on exceeds the
3032 // largest case in the switch.
3033 SDValue CMP = DAG.getSetCC(
3035 Sub.getValueType()),
3036 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
3037
3038 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3039 MVT::Other, CopyTo, CMP,
3040 DAG.getBasicBlock(JT.Default));
3041
3042 // Avoid emitting unnecessary branches to the next block.
3043 if (JT.MBB != NextBlock(SwitchBB))
3044 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
3045 DAG.getBasicBlock(JT.MBB));
3046
3047 DAG.setRoot(BrCond);
3048 } else {
3049 // Avoid emitting unnecessary branches to the next block.
3050 if (JT.MBB != NextBlock(SwitchBB))
3051 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
3052 DAG.getBasicBlock(JT.MBB)));
3053 else
3054 DAG.setRoot(CopyTo);
3055 }
3056}
3057
3058/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
3059/// variable if there exists one.
3061 SDValue &Chain) {
3062 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3063 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3064 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3068 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
3069 if (Global) {
3070 MachinePointerInfo MPInfo(Global);
3074 MPInfo, Flags, LocationSize::precise(PtrTy.getSizeInBits() / 8),
3075 DAG.getEVTAlign(PtrTy));
3076 DAG.setNodeMemRefs(Node, {MemRef});
3077 }
3078 if (PtrTy != PtrMemTy)
3079 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
3080 return SDValue(Node, 0);
3081}
3082
3083/// Codegen a new tail for a stack protector check ParentMBB which has had its
3084/// tail spliced into a stack protector check success bb.
3085///
3086/// For a high level explanation of how this fits into the stack protector
3087/// generation see the comment on the declaration of class
3088/// StackProtectorDescriptor.
3090 MachineBasicBlock *ParentBB) {
3091
3092 // First create the loads to the guard/stack slot for the comparison.
3094 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
3095 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
3096
3097 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
3098 int FI = MFI.getStackProtectorIndex();
3099
3100 SDValue Guard;
3101 SDLoc dl = getCurSDLoc();
3102 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
3103 const Module &M = *ParentBB->getParent()->getFunction().getParent();
3104 Align Align =
3105 DAG.getDataLayout().getPrefTypeAlign(PointerType::get(M.getContext(), 0));
3106
3107 // Generate code to load the content of the guard slot.
3108 SDValue GuardVal = DAG.getLoad(
3109 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
3112
3113 if (TLI.useStackGuardXorFP())
3114 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
3115
3116 // Retrieve guard check function, nullptr if instrumentation is inlined.
3117 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
3118 // The target provides a guard check function to validate the guard value.
3119 // Generate a call to that function with the content of the guard slot as
3120 // argument.
3121 FunctionType *FnTy = GuardCheckFn->getFunctionType();
3122 assert(FnTy->getNumParams() == 1 && "Invalid function signature");
3123
3126 Entry.Node = GuardVal;
3127 Entry.Ty = FnTy->getParamType(0);
3128 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
3129 Entry.IsInReg = true;
3130 Args.push_back(Entry);
3131
3135 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
3136 getValue(GuardCheckFn), std::move(Args));
3137
3138 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
3139 DAG.setRoot(Result.second);
3140 return;
3141 }
3142
3143 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
3144 // Otherwise, emit a volatile load to retrieve the stack guard value.
3145 SDValue Chain = DAG.getEntryNode();
3146 if (TLI.useLoadStackGuardNode(M)) {
3147 Guard = getLoadStackGuard(DAG, dl, Chain);
3148 } else {
3149 const Value *IRGuard = TLI.getSDagStackGuard(M);
3150 SDValue GuardPtr = getValue(IRGuard);
3151
3152 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
3153 MachinePointerInfo(IRGuard, 0), Align,
3155 }
3156
3157 // Perform the comparison via a getsetcc.
3159 *DAG.getContext(),
3160 Guard.getValueType()),
3161 Guard, GuardVal, ISD::SETNE);
3162
3163 // If the guard/stackslot do not equal, branch to failure MBB.
3164 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
3165 MVT::Other, GuardVal.getOperand(0),
3166 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
3167 // Otherwise branch to success MBB.
3168 SDValue Br = DAG.getNode(ISD::BR, dl,
3169 MVT::Other, BrCond,
3171
3172 DAG.setRoot(Br);
3173}
3174
3175/// Codegen the failure basic block for a stack protector check.
3176///
3177/// A failure stack protector machine basic block consists simply of a call to
3178/// __stack_chk_fail().
3179///
3180/// For a high level explanation of how this fits into the stack protector
3181/// generation see the comment on the declaration of class
3182/// StackProtectorDescriptor.
3183void
3187 CallOptions.setDiscardResult(true);
3188 SDValue Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL,
3189 MVT::isVoid, {}, CallOptions, getCurSDLoc())
3190 .second;
3191
3192 // Emit a trap instruction if we are required to do so.
3193 const TargetOptions &TargetOpts = DAG.getTarget().Options;
3194 if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)
3195 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
3196
3197 DAG.setRoot(Chain);
3198}
3199
3200/// visitBitTestHeader - This function emits necessary code to produce value
3201/// suitable for "bit tests"
3203 MachineBasicBlock *SwitchBB) {
3204 SDLoc dl = getCurSDLoc();
3205
3206 // Subtract the minimum value.
3207 SDValue SwitchOp = getValue(B.SValue);
3208 EVT VT = SwitchOp.getValueType();
3209 SDValue RangeSub =
3210 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
3211
3212 // Determine the type of the test operands.
3214 bool UsePtrType = false;
3215 if (!TLI.isTypeLegal(VT)) {
3216 UsePtrType = true;
3217 } else {
3218 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
3219 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
3220 // Switch table case range are encoded into series of masks.
3221 // Just use pointer type, it's guaranteed to fit.
3222 UsePtrType = true;
3223 break;
3224 }
3225 }
3226 SDValue Sub = RangeSub;
3227 if (UsePtrType) {
3228 VT = TLI.getPointerTy(DAG.getDataLayout());
3229 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
3230 }
3231
3232 B.RegVT = VT.getSimpleVT();
3233 B.Reg = FuncInfo.CreateReg(B.RegVT);
3234 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
3235
3236 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
3237
3238 if (!B.FallthroughUnreachable)
3239 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
3240 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
3241 SwitchBB->normalizeSuccProbs();
3242
3243 SDValue Root = CopyTo;
3244 if (!B.FallthroughUnreachable) {
3245 // Conditional branch to the default block.
3246 SDValue RangeCmp = DAG.getSetCC(dl,
3248 RangeSub.getValueType()),
3249 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
3250 ISD::SETUGT);
3251
3252 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
3253 DAG.getBasicBlock(B.Default));
3254 }
3255
3256 // Avoid emitting unnecessary branches to the next block.
3257 if (MBB != NextBlock(SwitchBB))
3258 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
3259
3260 DAG.setRoot(Root);
3261}
3262
3263/// visitBitTestCase - this function produces one "bit test"
3265 MachineBasicBlock *NextMBB,
3266 BranchProbability BranchProbToNext,
3267 Register Reg, BitTestCase &B,
3268 MachineBasicBlock *SwitchBB) {
3269 SDLoc dl = getCurSDLoc();
3270 MVT VT = BB.RegVT;
3271 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
3272 SDValue Cmp;
3273 unsigned PopCount = llvm::popcount(B.Mask);
3275 if (PopCount == 1) {
3276 // Testing for a single bit; just compare the shift count with what it
3277 // would need to be to shift a 1 bit in that position.
3278 Cmp = DAG.getSetCC(
3280 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT),
3281 ISD::SETEQ);
3282 } else if (PopCount == BB.Range) {
3283 // There is only one zero bit in the range, test for it directly.
3284 Cmp = DAG.getSetCC(
3286 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE);
3287 } else {
3288 // Make desired shift
3289 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
3290 DAG.getConstant(1, dl, VT), ShiftOp);
3291
3292 // Emit bit tests and jumps
3293 SDValue AndOp = DAG.getNode(ISD::AND, dl,
3294 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
3295 Cmp = DAG.getSetCC(
3297 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
3298 }
3299
3300 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
3301 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
3302 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
3303 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
3304 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
3305 // one as they are relative probabilities (and thus work more like weights),
3306 // and hence we need to normalize them to let the sum of them become one.
3307 SwitchBB->normalizeSuccProbs();
3308
3309 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
3310 MVT::Other, getControlRoot(),
3311 Cmp, DAG.getBasicBlock(B.TargetBB));
3312
3313 // Avoid emitting unnecessary branches to the next block.
3314 if (NextMBB != NextBlock(SwitchBB))
3315 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
3316 DAG.getBasicBlock(NextMBB));
3317
3318 DAG.setRoot(BrAnd);
3319}
3320
3321void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
3322 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
3323
3324 // Retrieve successors. Look through artificial IR level blocks like
3325 // catchswitch for successors.
3326 MachineBasicBlock *Return = FuncInfo.getMBB(I.getSuccessor(0));
3327 const BasicBlock *EHPadBB = I.getSuccessor(1);
3328 MachineBasicBlock *EHPadMBB = FuncInfo.getMBB(EHPadBB);
3329
3330 // Deopt and ptrauth bundles are lowered in helper functions, and we don't
3331 // have to do anything here to lower funclet bundles.
3332 assert(!I.hasOperandBundlesOtherThan(
3333 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
3334 LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
3335 LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
3336 LLVMContext::OB_clang_arc_attachedcall}) &&
3337 "Cannot lower invokes with arbitrary operand bundles yet!");
3338
3339 const Value *Callee(I.getCalledOperand());
3340 const Function *Fn = dyn_cast<Function>(Callee);
3341 if (isa<InlineAsm>(Callee))
3342 visitInlineAsm(I, EHPadBB);
3343 else if (Fn && Fn->isIntrinsic()) {
3344 switch (Fn->getIntrinsicID()) {
3345 default:
3346 llvm_unreachable("Cannot invoke this intrinsic");
3347 case Intrinsic::donothing:
3348 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
3349 case Intrinsic::seh_try_begin:
3350 case Intrinsic::seh_scope_begin:
3351 case Intrinsic::seh_try_end:
3352 case Intrinsic::seh_scope_end:
3353 if (EHPadMBB)
3354 // a block referenced by EH table
3355 // so dtor-funclet not removed by opts
3356 EHPadMBB->setMachineBlockAddressTaken();
3357 break;
3358 case Intrinsic::experimental_patchpoint_void:
3359 case Intrinsic::experimental_patchpoint:
3360 visitPatchpoint(I, EHPadBB);
3361 break;
3362 case Intrinsic::experimental_gc_statepoint:
3363 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB);
3364 break;
3365 case Intrinsic::wasm_rethrow: {
3366 // This is usually done in visitTargetIntrinsic, but this intrinsic is
3367 // special because it can be invoked, so we manually lower it to a DAG
3368 // node here.
3370 Ops.push_back(getControlRoot()); // inchain for the terminator node
3372 Ops.push_back(
3373 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(),
3375 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
3377 break;
3378 }
3379 }
3380 } else if (I.hasDeoptState()) {
3381 // Currently we do not lower any intrinsic calls with deopt operand bundles.
3382 // Eventually we will support lowering the @llvm.experimental.deoptimize
3383 // intrinsic, and right now there are no plans to support other intrinsics
3384 // with deopt state.
3385 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
3386 } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
3387 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), EHPadBB);
3388 } else {
3389 LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
3390 }
3391
3392 // If the value of the invoke is used outside of its defining block, make it
3393 // available as a virtual register.
3394 // We already took care of the exported value for the statepoint instruction
3395 // during call to the LowerStatepoint.
3396 if (!isa<GCStatepointInst>(I)) {
3398 }
3399
3402 BranchProbability EHPadBBProb =
3403 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
3405 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
3406
3407 // Update successor info.
3408 addSuccessorWithProb(InvokeMBB, Return);
3409 for (auto &UnwindDest : UnwindDests) {
3410 UnwindDest.first->setIsEHPad();
3411 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
3412 }
3413 InvokeMBB->normalizeSuccProbs();
3414
3415 // Drop into normal successor.
3417 DAG.getBasicBlock(Return)));
3418}
3419
3420void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
3421 MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
3422
3423 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
3424 // have to do anything here to lower funclet bundles.
3425 assert(!I.hasOperandBundlesOtherThan(
3426 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
3427 "Cannot lower callbrs with arbitrary operand bundles yet!");
3428
3429 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr");
3430 visitInlineAsm(I);
3432
3433 // Retrieve successors.
3435 Dests.insert(I.getDefaultDest());
3436 MachineBasicBlock *Return = FuncInfo.getMBB(I.getDefaultDest());
3437
3438 // Update successor info.
3439 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
3440 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
3441 BasicBlock *Dest = I.getIndirectDest(i);
3443 Target->setIsInlineAsmBrIndirectTarget();
3444 Target->setMachineBlockAddressTaken();
3445 Target->setLabelMustBeEmitted();
3446 // Don't add duplicate machine successors.
3447 if (Dests.insert(Dest).second)
3448 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
3449 }
3450 CallBrMBB->normalizeSuccProbs();
3451
3452 // Drop into default successor.
3454 MVT::Other, getControlRoot(),
3455 DAG.getBasicBlock(Return)));
3456}
3457
3458void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
3459 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
3460}
3461
3462void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
3464 "Call to landingpad not in landing pad!");
3465
3466 // If there aren't registers to copy the values into (e.g., during SjLj
3467 // exceptions), then don't bother to create these DAG nodes.
3469 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
3470 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
3471 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
3472 return;
3473
3474 // If landingpad's return type is token type, we don't create DAG nodes
3475 // for its exception pointer and selector value. The extraction of exception
3476 // pointer or selector value from token type landingpads is not currently
3477 // supported.
3478 if (LP.getType()->isTokenTy())
3479 return;
3480
3481 SmallVector<EVT, 2> ValueVTs;
3482 SDLoc dl = getCurSDLoc();
3483 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
3484 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
3485
3486 // Get the two live-in registers as SDValues. The physregs have already been
3487 // copied into virtual registers.
3488 SDValue Ops[2];
3490 Ops[0] = DAG.getZExtOrTrunc(
3494 dl, ValueVTs[0]);
3495 } else {
3496 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
3497 }
3498 Ops[1] = DAG.getZExtOrTrunc(
3502 dl, ValueVTs[1]);
3503
3504 // Merge into one.
3506 DAG.getVTList(ValueVTs), Ops);
3507 setValue(&LP, Res);
3508}
3509
3512 // Update JTCases.
3513 for (JumpTableBlock &JTB : SL->JTCases)
3514 if (JTB.first.HeaderBB == First)
3515 JTB.first.HeaderBB = Last;
3516
3517 // Update BitTestCases.
3518 for (BitTestBlock &BTB : SL->BitTestCases)
3519 if (BTB.Parent == First)
3520 BTB.Parent = Last;
3521}
3522
3523void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
3524 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
3525
3526 // Update machine-CFG edges with unique successors.
3528 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
3529 BasicBlock *BB = I.getSuccessor(i);
3530 bool Inserted = Done.insert(BB).second;
3531 if (!Inserted)
3532 continue;
3533
3534 MachineBasicBlock *Succ = FuncInfo.getMBB(BB);
3535 addSuccessorWithProb(IndirectBrMBB, Succ);
3536 }
3537 IndirectBrMBB->normalizeSuccProbs();
3538
3540 MVT::Other, getControlRoot(),
3541 getValue(I.getAddress())));
3542}
3543
3544void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
3546 return;
3547
3548 // We may be able to ignore unreachable behind a noreturn call.
3549 if (const CallInst *Call = dyn_cast_or_null<CallInst>(I.getPrevNode());
3550 Call && Call->doesNotReturn()) {
3552 return;
3553 // Do not emit an additional trap instruction.
3554 if (Call->isNonContinuableTrap())
3555 return;
3556 }
3557
3558 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
3559}
3560
3561void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3563 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3564 Flags.copyFMF(*FPOp);
3565
3566 SDValue Op = getValue(I.getOperand(0));
3567 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3568 Op, Flags);
3569 setValue(&I, UnNodeValue);
3570}
3571
3572void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3574 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3575 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3576 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3577 }
3578 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I))
3579 Flags.setExact(ExactOp->isExact());
3580 if (auto *DisjointOp = dyn_cast<PossiblyDisjointInst>(&I))
3581 Flags.setDisjoint(DisjointOp->isDisjoint());
3582 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3583 Flags.copyFMF(*FPOp);
3584
3585 SDValue Op1 = getValue(I.getOperand(0));
3586 SDValue Op2 = getValue(I.getOperand(1));
3587 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3588 Op1, Op2, Flags);
3589 setValue(&I, BinNodeValue);
3590}
3591
3592void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3593 SDValue Op1 = getValue(I.getOperand(0));
3594 SDValue Op2 = getValue(I.getOperand(1));
3595
3597 Op1.getValueType(), DAG.getDataLayout());
3598
3599 // Coerce the shift amount to the right type if we can. This exposes the
3600 // truncate or zext to optimization early.
3601 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
3603 "Unexpected shift type");
3604 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy);
3605 }
3606
3607 bool nuw = false;
3608 bool nsw = false;
3609 bool exact = false;
3610
3611 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
3612
3613 if (const OverflowingBinaryOperator *OFBinOp =
3614 dyn_cast<const OverflowingBinaryOperator>(&I)) {
3615 nuw = OFBinOp->hasNoUnsignedWrap();
3616 nsw = OFBinOp->hasNoSignedWrap();
3617 }
3618 if (const PossiblyExactOperator *ExactOp =
3619 dyn_cast<const PossiblyExactOperator>(&I))
3620 exact = ExactOp->isExact();
3621 }
3623 Flags.setExact(exact);
3624 Flags.setNoSignedWrap(nsw);
3625 Flags.setNoUnsignedWrap(nuw);
3626 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3627 Flags);
3628 setValue(&I, Res);
3629}
3630
3631void SelectionDAGBuilder::visitSDiv(const User &I) {
3632 SDValue Op1 = getValue(I.getOperand(0));
3633 SDValue Op2 = getValue(I.getOperand(1));
3634
3636 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3637 cast<PossiblyExactOperator>(&I)->isExact());
3639 Op2, Flags));
3640}
3641
3642void SelectionDAGBuilder::visitICmp(const ICmpInst &I) {
3643 ICmpInst::Predicate predicate = I.getPredicate();
3644 SDValue Op1 = getValue(I.getOperand(0));
3645 SDValue Op2 = getValue(I.getOperand(1));
3646 ISD::CondCode Opcode = getICmpCondCode(predicate);
3647
3648 auto &TLI = DAG.getTargetLoweringInfo();
3649 EVT MemVT =
3650 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3651
3652 // If a pointer's DAG type is larger than its memory type then the DAG values
3653 // are zero-extended. This breaks signed comparisons so truncate back to the
3654 // underlying type before doing the compare.
3655 if (Op1.getValueType() != MemVT) {
3656 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3657 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3658 }
3659
3661 Flags.setSameSign(I.hasSameSign());
3662 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3663
3665 I.getType());
3666 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3667}
3668
3669void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) {
3670 FCmpInst::Predicate predicate = I.getPredicate();
3671 SDValue Op1 = getValue(I.getOperand(0));
3672 SDValue Op2 = getValue(I.getOperand(1));
3673
3674 ISD::CondCode Condition = getFCmpCondCode(predicate);
3675 auto *FPMO = cast<FPMathOperator>(&I);
3676 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath)
3677 Condition = getFCmpCodeWithoutNaN(Condition);
3678
3680 Flags.copyFMF(*FPMO);
3681 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
3682
3684 I.getType());
3685 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3686}
3687
3688// Check if the condition of the select has one use or two users that are both
3689// selects with the same condition.
3690static bool hasOnlySelectUsers(const Value *Cond) {
3691 return llvm::all_of(Cond->users(), [](const Value *V) {
3692 return isa<SelectInst>(V);
3693 });
3694}
3695
3696void SelectionDAGBuilder::visitSelect(const User &I) {
3697 SmallVector<EVT, 4> ValueVTs;
3699 ValueVTs);
3700 unsigned NumValues = ValueVTs.size();
3701 if (NumValues == 0) return;
3702
3703 SmallVector<SDValue, 4> Values(NumValues);
3704 SDValue Cond = getValue(I.getOperand(0));
3705 SDValue LHSVal = getValue(I.getOperand(1));
3706 SDValue RHSVal = getValue(I.getOperand(2));
3707 SmallVector<SDValue, 1> BaseOps(1, Cond);
3709 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
3710
3711 bool IsUnaryAbs = false;
3712 bool Negate = false;
3713
3715 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
3716 Flags.copyFMF(*FPOp);
3717
3718 Flags.setUnpredictable(
3719 cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3720
3721 // Min/max matching is only viable if all output VTs are the same.
3722 if (all_equal(ValueVTs)) {
3723 EVT VT = ValueVTs[0];
3724 LLVMContext &Ctx = *DAG.getContext();
3725 auto &TLI = DAG.getTargetLoweringInfo();
3726
3727 // We care about the legality of the operation after it has been type
3728 // legalized.
3729 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal)
3730 VT = TLI.getTypeToTransformTo(Ctx, VT);
3731
3732 // If the vselect is legal, assume we want to leave this as a vector setcc +
3733 // vselect. Otherwise, if this is going to be scalarized, we want to see if
3734 // min/max is legal on the scalar type.
3735 bool UseScalarMinMax = VT.isVector() &&
3737
3738 // ValueTracking's select pattern matching does not account for -0.0,
3739 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that
3740 // -0.0 is less than +0.0.
3741 const Value *LHS, *RHS;
3742 auto SPR = matchSelectPattern(&I, LHS, RHS);
3744 switch (SPR.Flavor) {
3745 case SPF_UMAX: Opc = ISD::UMAX; break;
3746 case SPF_UMIN: Opc = ISD::UMIN; break;
3747 case SPF_SMAX: Opc = ISD::SMAX; break;
3748 case SPF_SMIN: Opc = ISD::SMIN; break;
3749 case SPF_FMINNUM:
3750 switch (SPR.NaNBehavior) {
3751 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3752 case SPNB_RETURNS_NAN: break;
3753 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
3754 case SPNB_RETURNS_ANY:
3756 (UseScalarMinMax &&
3758 Opc = ISD::FMINNUM;
3759 break;
3760 }
3761 break;
3762 case SPF_FMAXNUM:
3763 switch (SPR.NaNBehavior) {
3764 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?");
3765 case SPNB_RETURNS_NAN: break;
3766 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
3767 case SPNB_RETURNS_ANY:
3769 (UseScalarMinMax &&
3771 Opc = ISD::FMAXNUM;
3772 break;
3773 }
3774 break;
3775 case SPF_NABS:
3776 Negate = true;
3777 [[fallthrough]];
3778 case SPF_ABS:
3779 IsUnaryAbs = true;
3780 Opc = ISD::ABS;
3781 break;
3782 default: break;
3783 }
3784
3785 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE &&
3786 (TLI.isOperationLegalOrCustom(Opc, VT) ||
3787 (UseScalarMinMax &&
3788 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3789 // If the underlying comparison instruction is used by any other
3790 // instruction, the consumed instructions won't be destroyed, so it is
3791 // not profitable to convert to a min/max.
3792 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3793 OpCode = Opc;
3794 LHSVal = getValue(LHS);
3795 RHSVal = getValue(RHS);
3796 BaseOps.clear();
3797 }
3798
3799 if (IsUnaryAbs) {
3800 OpCode = Opc;
3801 LHSVal = getValue(LHS);
3802 BaseOps.clear();
3803 }
3804 }
3805
3806 if (IsUnaryAbs) {
3807 for (unsigned i = 0; i != NumValues; ++i) {
3808 SDLoc dl = getCurSDLoc();
3809 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i);
3810 Values[i] =
3811 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i));
3812 if (Negate)
3813 Values[i] = DAG.getNegative(Values[i], dl, VT);
3814 }
3815 } else {
3816 for (unsigned i = 0; i != NumValues; ++i) {
3817 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3818 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3819 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3820 Values[i] = DAG.getNode(
3821 OpCode, getCurSDLoc(),
3822 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags);
3823 }
3824 }
3825
3827 DAG.getVTList(ValueVTs), Values));
3828}
3829
3830void SelectionDAGBuilder::visitTrunc(const User &I) {
3831 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3832 SDValue N = getValue(I.getOperand(0));
3834 I.getType());
3836 if (auto *Trunc = dyn_cast<TruncInst>(&I)) {
3837 Flags.setNoSignedWrap(Trunc->hasNoSignedWrap());
3838 Flags.setNoUnsignedWrap(Trunc->hasNoUnsignedWrap());
3839 }
3840
3841 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N, Flags));
3842}
3843
3844void SelectionDAGBuilder::visitZExt(const User &I) {
3845 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3846 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3847 SDValue N = getValue(I.getOperand(0));
3848 auto &TLI = DAG.getTargetLoweringInfo();
3849 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3850
3852 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3853 Flags.setNonNeg(PNI->hasNonNeg());
3854
3855 // Eagerly use nonneg information to canonicalize towards sign_extend if
3856 // that is the target's preference.
3857 // TODO: Let the target do this later.
3858 if (Flags.hasNonNeg() &&
3859 TLI.isSExtCheaperThanZExt(N.getValueType(), DestVT)) {
3861 return;
3862 }
3863
3864 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N, Flags));
3865}
3866
3867void SelectionDAGBuilder::visitSExt(const User &I) {
3868 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3869 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3870 SDValue N = getValue(I.getOperand(0));
3872 I.getType());
3874}
3875
3876void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3877 // FPTrunc is never a no-op cast, no need to check
3878 SDValue N = getValue(I.getOperand(0));
3879 SDLoc dl = getCurSDLoc();
3881 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3882 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3884 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3885}
3886
3887void SelectionDAGBuilder::visitFPExt(const User &I) {
3888 // FPExt is never a no-op cast, no need to check
3889 SDValue N = getValue(I.getOperand(0));
3891 I.getType());
3893}
3894
3895void SelectionDAGBuilder::visitFPToUI(const User &I) {
3896 // FPToUI is never a no-op cast, no need to check
3897 SDValue N = getValue(I.getOperand(0));
3899 I.getType());
3901}
3902
3903void SelectionDAGBuilder::visitFPToSI(const User &I) {
3904 // FPToSI is never a no-op cast, no need to check
3905 SDValue N = getValue(I.getOperand(0));
3907 I.getType());
3909}
3910
3911void SelectionDAGBuilder::visitUIToFP(const User &I) {
3912 // UIToFP is never a no-op cast, no need to check
3913 SDValue N = getValue(I.getOperand(0));
3915 I.getType());
3917 if (auto *PNI = dyn_cast<PossiblyNonNegInst>(&I))
3918 Flags.setNonNeg(PNI->hasNonNeg());
3919
3920 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N, Flags));
3921}
3922
3923void SelectionDAGBuilder::visitSIToFP(const User &I) {
3924 // SIToFP is never a no-op cast, no need to check
3925 SDValue N = getValue(I.getOperand(0));
3927 I.getType());
3929}
3930
3931void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3932 // What to do depends on the size of the integer and the size of the pointer.
3933 // We can either truncate, zero extend, or no-op, accordingly.
3934 SDValue N = getValue(I.getOperand(0));
3935 auto &TLI = DAG.getTargetLoweringInfo();
3937 I.getType());
3938 EVT PtrMemVT =
3939 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3940 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3941 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3942 setValue(&I, N);
3943}
3944
3945void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3946 // What to do depends on the size of the integer and the size of the pointer.
3947 // We can either truncate, zero extend, or no-op, accordingly.
3948 SDValue N = getValue(I.getOperand(0));
3949 auto &TLI = DAG.getTargetLoweringInfo();
3950 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3951 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3952 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3953 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3954 setValue(&I, N);
3955}
3956
3957void SelectionDAGBuilder::visitBitCast(const User &I) {
3958 SDValue N = getValue(I.getOperand(0));
3959 SDLoc dl = getCurSDLoc();
3961 I.getType());
3962
3963 // BitCast assures us that source and destination are the same size so this is
3964 // either a BITCAST or a no-op.
3965 if (DestVT != N.getValueType())
3967 DestVT, N)); // convert types.
3968 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3969 // might fold any kind of constant expression to an integer constant and that
3970 // is not what we are looking for. Only recognize a bitcast of a genuine
3971 // constant integer as an opaque constant.
3972 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3973 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3974 /*isOpaque*/true));
3975 else
3976 setValue(&I, N); // noop cast.
3977}
3978
3979void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3981 const Value *SV = I.getOperand(0);
3982 SDValue N = getValue(SV);
3983 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3984
3985 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3986 unsigned DestAS = I.getType()->getPointerAddressSpace();
3987
3988 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS))
3989 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3990
3991 setValue(&I, N);
3992}
3993
3994void SelectionDAGBuilder::visitInsertElement(const User &I) {
3996 SDValue InVec = getValue(I.getOperand(0));
3997 SDValue InVal = getValue(I.getOperand(1));
3998 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
4001 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4002 InVec, InVal, InIdx));
4003}
4004
4005void SelectionDAGBuilder::visitExtractElement(const User &I) {
4007 SDValue InVec = getValue(I.getOperand(0));
4008 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
4011 TLI.getValueType(DAG.getDataLayout(), I.getType()),
4012 InVec, InIdx));
4013}
4014
4015void SelectionDAGBuilder::visitShuffleVector(const User &I) {
4016 SDValue Src1 = getValue(I.getOperand(0));
4017 SDValue Src2 = getValue(I.getOperand(1));
4019 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I))
4020 Mask = SVI->getShuffleMask();
4021 else
4022 Mask = cast<ConstantExpr>(I).getShuffleMask();
4023 SDLoc DL = getCurSDLoc();
4025 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4026 EVT SrcVT = Src1.getValueType();
4027
4028 if (all_of(Mask, [](int Elem) { return Elem == 0; }) &&
4029 VT.isScalableVector()) {
4030 // Canonical splat form of first element of first input vector.
4031 SDValue FirstElt =
4034 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt));
4035 return;
4036 }
4037
4038 // For now, we only handle splats for scalable vectors.
4039 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation
4040 // for targets that support a SPLAT_VECTOR for non-scalable vector types.
4041 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle");
4042
4043 unsigned SrcNumElts = SrcVT.getVectorNumElements();
4044 unsigned MaskNumElts = Mask.size();
4045
4046 if (SrcNumElts == MaskNumElts) {
4047 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
4048 return;
4049 }
4050
4051 // Normalize the shuffle vector since mask and vector length don't match.
4052 if (SrcNumElts < MaskNumElts) {
4053 // Mask is longer than the source vectors. We can use concatenate vector to
4054 // make the mask and vectors lengths match.
4055
4056 if (MaskNumElts % SrcNumElts == 0) {
4057 // Mask length is a multiple of the source vector length.
4058 // Check if the shuffle is some kind of concatenation of the input
4059 // vectors.
4060 unsigned NumConcat = MaskNumElts / SrcNumElts;
4061 bool IsConcat = true;
4062 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
4063 for (unsigned i = 0; i != MaskNumElts; ++i) {
4064 int Idx = Mask[i];
4065 if (Idx < 0)
4066 continue;
4067 // Ensure the indices in each SrcVT sized piece are sequential and that
4068 // the same source is used for the whole piece.
4069 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
4070 (ConcatSrcs[i / SrcNumElts] >= 0 &&
4071 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
4072 IsConcat = false;
4073 break;
4074 }
4075 // Remember which source this index came from.
4076 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
4077 }
4078
4079 // The shuffle is concatenating multiple vectors together. Just emit
4080 // a CONCAT_VECTORS operation.
4081 if (IsConcat) {
4082 SmallVector<SDValue, 8> ConcatOps;
4083 for (auto Src : ConcatSrcs) {
4084 if (Src < 0)
4085 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
4086 else if (Src == 0)
4087 ConcatOps.push_back(Src1);
4088 else
4089 ConcatOps.push_back(Src2);
4090 }
4091 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
4092 return;
4093 }
4094 }
4095
4096 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
4097 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
4098 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
4099 PaddedMaskNumElts);
4100
4101 // Pad both vectors with undefs to make them the same length as the mask.
4102 SDValue UndefVal = DAG.getUNDEF(SrcVT);
4103
4104 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
4105 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
4106 MOps1[0] = Src1;
4107 MOps2[0] = Src2;
4108
4109 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
4110 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
4111
4112 // Readjust mask for new input vector length.
4113 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
4114 for (unsigned i = 0; i != MaskNumElts; ++i) {
4115 int Idx = Mask[i];
4116 if (Idx >= (int)SrcNumElts)
4117 Idx -= SrcNumElts - PaddedMaskNumElts;
4118 MappedOps[i] = Idx;
4119 }
4120
4121 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
4122
4123 // If the concatenated vector was padded, extract a subvector with the
4124 // correct number of elements.
4125 if (MaskNumElts != PaddedMaskNumElts)
4128
4129 setValue(&I, Result);
4130 return;
4131 }
4132
4133 assert(SrcNumElts > MaskNumElts);
4134
4135 // Analyze the access pattern of the vector to see if we can extract
4136 // two subvectors and do the shuffle.
4137 int StartIdx[2] = {-1, -1}; // StartIdx to extract from
4138 bool CanExtract = true;
4139 for (int Idx : Mask) {
4140 unsigned Input = 0;
4141 if (Idx < 0)
4142 continue;
4143
4144 if (Idx >= (int)SrcNumElts) {
4145 Input = 1;
4146 Idx -= SrcNumElts;
4147 }
4148
4149 // If all the indices come from the same MaskNumElts sized portion of
4150 // the sources we can use extract. Also make sure the extract wouldn't
4151 // extract past the end of the source.
4152 int NewStartIdx = alignDown(Idx, MaskNumElts);
4153 if (NewStartIdx + MaskNumElts > SrcNumElts ||
4154 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
4155 CanExtract = false;
4156 // Make sure we always update StartIdx as we use it to track if all
4157 // elements are undef.
4158 StartIdx[Input] = NewStartIdx;
4159 }
4160
4161 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
4162 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
4163 return;
4164 }
4165 if (CanExtract) {
4166 // Extract appropriate subvector and generate a vector shuffle
4167 for (unsigned Input = 0; Input < 2; ++Input) {
4168 SDValue &Src = Input == 0 ? Src1 : Src2;
4169 if (StartIdx[Input] < 0)
4170 Src = DAG.getUNDEF(VT);
4171 else {
4172 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
4173 DAG.getVectorIdxConstant(StartIdx[Input], DL));
4174 }
4175 }
4176
4177 // Calculate new mask.
4178 SmallVector<int, 8> MappedOps(Mask);
4179 for (int &Idx : MappedOps) {
4180 if (Idx >= (int)SrcNumElts)
4181 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
4182 else if (Idx >= 0)
4183 Idx -= StartIdx[0];
4184 }
4185
4186 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
4187 return;
4188 }
4189
4190 // We can't use either concat vectors or extract subvectors so fall back to
4191 // replacing the shuffle with extract and build vector.
4192 // to insert and build vector.
4193 EVT EltVT = VT.getVectorElementType();
4195 for (int Idx : Mask) {
4196 SDValue Res;
4197
4198 if (Idx < 0) {
4199 Res = DAG.getUNDEF(EltVT);
4200 } else {
4201 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
4202 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
4203
4204 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src,
4206 }
4207
4208 Ops.push_back(Res);
4209 }
4210
4211 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
4212}
4213
4214void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) {
4215 ArrayRef<unsigned> Indices = I.getIndices();
4216 const Value *Op0 = I.getOperand(0);
4217 const Value *Op1 = I.getOperand(1);
4218 Type *AggTy = I.getType();
4219 Type *ValTy = Op1->getType();
4220 bool IntoUndef = isa<UndefValue>(Op0);
4221 bool FromUndef = isa<UndefValue>(Op1);
4222
4223 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4224
4226 SmallVector<EVT, 4> AggValueVTs;
4227 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
4228 SmallVector<EVT, 4> ValValueVTs;
4229 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4230
4231 unsigned NumAggValues = AggValueVTs.size();
4232 unsigned NumValValues = ValValueVTs.size();
4233 SmallVector<SDValue, 4> Values(NumAggValues);
4234
4235 // Ignore an insertvalue that produces an empty object
4236 if (!NumAggValues) {
4237 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4238 return;
4239 }
4240
4241 SDValue Agg = getValue(Op0);
4242 unsigned i = 0;
4243 // Copy the beginning value(s) from the original aggregate.
4244 for (; i != LinearIndex; ++i)
4245 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4246 SDValue(Agg.getNode(), Agg.getResNo() + i);
4247 // Copy values from the inserted value(s).
4248 if (NumValValues) {
4249 SDValue Val = getValue(Op1);
4250 for (; i != LinearIndex + NumValValues; ++i)
4251 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4252 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
4253 }
4254 // Copy remaining value(s) from the original aggregate.
4255 for (; i != NumAggValues; ++i)
4256 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
4257 SDValue(Agg.getNode(), Agg.getResNo() + i);
4258
4260 DAG.getVTList(AggValueVTs), Values));
4261}
4262
4263void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) {
4264 ArrayRef<unsigned> Indices = I.getIndices();
4265 const Value *Op0 = I.getOperand(0);
4266 Type *AggTy = Op0->getType();
4267 Type *ValTy = I.getType();
4268 bool OutOfUndef = isa<UndefValue>(Op0);
4269
4270 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
4271
4273 SmallVector<EVT, 4> ValValueVTs;
4274 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
4275
4276 unsigned NumValValues = ValValueVTs.size();
4277
4278 // Ignore a extractvalue that produces an empty object
4279 if (!NumValValues) {
4280 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
4281 return;
4282 }
4283
4284 SmallVector<SDValue, 4> Values(NumValValues);
4285
4286 SDValue Agg = getValue(Op0);
4287 // Copy out the selected value(s).
4288 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
4289 Values[i - LinearIndex] =
4290 OutOfUndef ?
4291 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
4292 SDValue(Agg.getNode(), Agg.getResNo() + i);
4293
4295 DAG.getVTList(ValValueVTs), Values));
4296}
4297
4298void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
4299 Value *Op0 = I.getOperand(0);
4300 // Note that the pointer operand may be a vector of pointers. Take the scalar
4301 // element which holds a pointer.
4302 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
4303 SDValue N = getValue(Op0);
4304 SDLoc dl = getCurSDLoc();
4305 auto &TLI = DAG.getTargetLoweringInfo();
4306 GEPNoWrapFlags NW = cast<GEPOperator>(I).getNoWrapFlags();
4307
4308 // Normalize Vector GEP - all scalar operands should be converted to the
4309 // splat vector.
4310 bool IsVectorGEP = I.getType()->isVectorTy();
4311 ElementCount VectorElementCount =
4312 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount()
4314
4315 if (IsVectorGEP && !N.getValueType().isVector()) {
4317 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount);
4318 N = DAG.getSplat(VT, dl, N);
4319 }
4320
4321 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
4322 GTI != E; ++GTI) {
4323 const Value *Idx = GTI.getOperand();
4324 if (StructType *StTy = GTI.getStructTypeOrNull()) {
4325 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
4326 if (Field) {
4327 // N = N + Offset
4330
4331 // In an inbounds GEP with an offset that is nonnegative even when
4332 // interpreted as signed, assume there is no unsigned overflow.
4334 if (NW.hasNoUnsignedWrap() ||
4335 (int64_t(Offset) >= 0 && NW.hasNoUnsignedSignedWrap()))
4337
4338 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
4339 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
4340 }
4341 } else {
4342 // IdxSize is the width of the arithmetic according to IR semantics.
4343 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth
4344 // (and fix up the result later).
4345 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
4346 MVT IdxTy = MVT::getIntegerVT(IdxSize);
4347 TypeSize ElementSize =
4348 GTI.getSequentialElementStride(DAG.getDataLayout());
4349 // We intentionally mask away the high bits here; ElementSize may not
4350 // fit in IdxTy.
4351 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue(),
4352 /*isSigned=*/false, /*implicitTrunc=*/true);
4353 bool ElementScalable = ElementSize.isScalable();
4354
4355 // If this is a scalar constant or a splat vector of constants,
4356 // handle it quickly.
4357 const auto *C = dyn_cast<Constant>(Idx);
4358 if (C && isa<VectorType>(C->getType()))
4359 C = C->getSplatValue();
4360
4361 const auto *CI = dyn_cast_or_null<ConstantInt>(C);
4362 if (CI && CI->isZero())
4363 continue;
4364 if (CI && !ElementScalable) {
4365 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize);
4367 SDValue OffsVal;
4368 if (IsVectorGEP)
4369 OffsVal = DAG.getConstant(
4370 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount));
4371 else
4372 OffsVal = DAG.getConstant(Offs, dl, IdxTy);
4373
4374 // In an inbounds GEP with an offset that is nonnegative even when
4375 // interpreted as signed, assume there is no unsigned overflow.
4377 if (NW.hasNoUnsignedWrap() ||
4378 (Offs.isNonNegative() && NW.hasNoUnsignedSignedWrap()))
4379 Flags.setNoUnsignedWrap(true);
4380
4381 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
4382
4383 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
4384 continue;
4385 }
4386
4387 // N = N + Idx * ElementMul;
4388 SDValue IdxN = getValue(Idx);
4389
4390 if (!IdxN.getValueType().isVector() && IsVectorGEP) {
4392 VectorElementCount);
4393 IdxN = DAG.getSplat(VT, dl, IdxN);
4394 }
4395
4396 // If the index is smaller or larger than intptr_t, truncate or extend
4397 // it.
4398 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
4399
4400 SDNodeFlags ScaleFlags;
4401 // The multiplication of an index by the type size does not wrap the
4402 // pointer index type in a signed sense (mul nsw).
4404
4405 // The multiplication of an index by the type size does not wrap the
4406 // pointer index type in an unsigned sense (mul nuw).
4407 ScaleFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4408
4409 if (ElementScalable) {
4410 EVT VScaleTy = N.getValueType().getScalarType();
4411 SDValue VScale = DAG.getNode(
4412 ISD::VSCALE, dl, VScaleTy,
4413 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy));
4414 if (IsVectorGEP)
4415 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale);
4416 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale,
4417 ScaleFlags);
4418 } else {
4419 // If this is a multiply by a power of two, turn it into a shl
4420 // immediately. This is a very common case.
4421 if (ElementMul != 1) {
4422 if (ElementMul.isPowerOf2()) {
4423 unsigned Amt = ElementMul.logBase2();
4424 IdxN = DAG.getNode(ISD::SHL, dl, N.getValueType(), IdxN,
4425 DAG.getConstant(Amt, dl, IdxN.getValueType()),
4426 ScaleFlags);
4427 } else {
4428 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl,
4429 IdxN.getValueType());
4430 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, Scale,
4431 ScaleFlags);
4432 }
4433 }
4434 }
4435
4436 // The successive addition of the current address, truncated to the
4437 // pointer index type and interpreted as an unsigned number, and each
4438 // offset, also interpreted as an unsigned number, does not wrap the
4439 // pointer index type (add nuw).
4440 SDNodeFlags AddFlags;
4441 AddFlags.setNoUnsignedWrap(NW.hasNoUnsignedWrap());
4442
4443 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, IdxN, AddFlags);
4444 }
4445 }
4446
4447 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
4448 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
4449 if (IsVectorGEP) {
4450 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount);
4451 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount);
4452 }
4453
4454 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds())
4455 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
4456
4457 setValue(&I, N);
4458}
4459
4460void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
4461 // If this is a fixed sized alloca in the entry block of the function,
4462 // allocate it statically on the stack.
4463 if (FuncInfo.StaticAllocaMap.count(&I))
4464 return; // getValue will auto-populate this.
4465
4466 SDLoc dl = getCurSDLoc();
4467 Type *Ty = I.getAllocatedType();
4469 auto &DL = DAG.getDataLayout();
4470 TypeSize TySize = DL.getTypeAllocSize(Ty);
4471 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign());
4472
4473 SDValue AllocSize = getValue(I.getArraySize());
4474
4475 EVT IntPtr = TLI.getPointerTy(DL, I.getAddressSpace());
4476 if (AllocSize.getValueType() != IntPtr)
4477 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
4478
4479 if (TySize.isScalable())
4480 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4481 DAG.getVScale(dl, IntPtr,
4482 APInt(IntPtr.getScalarSizeInBits(),
4483 TySize.getKnownMinValue())));
4484 else {
4485 SDValue TySizeValue =
4487 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize,
4488 DAG.getZExtOrTrunc(TySizeValue, dl, IntPtr));
4489 }
4490
4491 // Handle alignment. If the requested alignment is less than or equal to
4492 // the stack alignment, ignore it. If the size is greater than or equal to
4493 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
4495 if (*Alignment <= StackAlign)
4496 Alignment = std::nullopt;
4497
4498 const uint64_t StackAlignMask = StackAlign.value() - 1U;
4499 // Round the size of the allocation up to the stack alignment size
4500 // by add SA-1 to the size. This doesn't overflow because we're computing
4501 // an address inside an alloca.
4502 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
4503 DAG.getConstant(StackAlignMask, dl, IntPtr),
4505
4506 // Mask out the low bits for alignment purposes.
4507 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
4508 DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
4509
4510 SDValue Ops[] = {
4511 getRoot(), AllocSize,
4512 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)};
4513 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
4514 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
4515 setValue(&I, DSA);
4516 DAG.setRoot(DSA.getValue(1));
4517
4519}
4520
4521static const MDNode *getRangeMetadata(const Instruction &I) {
4522 // If !noundef is not present, then !range violation results in a poison
4523 // value rather than immediate undefined behavior. In theory, transferring
4524 // these annotations to SDAG is fine, but in practice there are key SDAG
4525 // transforms that are known not to be poison-safe, such as folding logical
4526 // and/or to bitwise and/or. For now, only transfer !range if !noundef is
4527 // also present.
4528 if (!I.hasMetadata(LLVMContext::MD_noundef))
4529 return nullptr;
4530 return I.getMetadata(LLVMContext::MD_range);
4531}
4532
4533static std::optional<ConstantRange> getRange(const Instruction &I) {
4534 if (const auto *CB = dyn_cast<CallBase>(&I)) {
4535 // see comment in getRangeMetadata about this check
4536 if (CB->hasRetAttr(Attribute::NoUndef))
4537 return CB->getRange();
4538 }
4539 if (const MDNode *Range = getRangeMetadata(I))
4541 return std::nullopt;
4542}
4543
4544void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
4545 if (I.isAtomic())
4546 return visitAtomicLoad(I);
4547
4549 const Value *SV = I.getOperand(0);
4550 if (TLI.supportSwiftError()) {
4551 // Swifterror values can come from either a function parameter with
4552 // swifterror attribute or an alloca with swifterror attribute.
4553 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
4554 if (Arg->hasSwiftErrorAttr())
4555 return visitLoadFromSwiftError(I);
4556 }
4557
4558 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
4559 if (Alloca->isSwiftError())
4560 return visitLoadFromSwiftError(I);
4561 }
4562 }
4563
4564 SDValue Ptr = getValue(SV);
4565
4566 Type *Ty = I.getType();
4567 SmallVector<EVT, 4> ValueVTs, MemVTs;
4569 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4570 unsigned NumValues = ValueVTs.size();
4571 if (NumValues == 0)
4572 return;
4573
4574 Align Alignment = I.getAlign();
4575 AAMDNodes AAInfo = I.getAAMetadata();
4576 const MDNode *Ranges = getRangeMetadata(I);
4577 bool isVolatile = I.isVolatile();
4578 MachineMemOperand::Flags MMOFlags =
4580
4581 SDValue Root;
4582 bool ConstantMemory = false;
4583 if (isVolatile)
4584 // Serialize volatile loads with other side effects.
4585 Root = getRoot();
4586 else if (NumValues > MaxParallelChains)
4587 Root = getMemoryRoot();
4588 else if (AA &&
4590 SV,
4592 AAInfo))) {
4593 // Do not serialize (non-volatile) loads of constant memory with anything.
4594 Root = DAG.getEntryNode();
4595 ConstantMemory = true;
4597 } else {
4598 // Do not serialize non-volatile loads against each other.
4599 Root = DAG.getRoot();
4600 }
4601
4602 SDLoc dl = getCurSDLoc();
4603
4604 if (isVolatile)
4605 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4606
4607 SmallVector<SDValue, 4> Values(NumValues);
4608 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4609
4610 unsigned ChainI = 0;
4611 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4612 // Serializing loads here may result in excessive register pressure, and
4613 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4614 // could recover a bit by hoisting nodes upward in the chain by recognizing
4615 // they are side-effect free or do not alias. The optimizer should really
4616 // avoid this case by converting large object/array copies to llvm.memcpy
4617 // (MaxParallelChains should always remain as failsafe).
4618 if (ChainI == MaxParallelChains) {
4619 assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4620 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4621 ArrayRef(Chains.data(), ChainI));
4622 Root = Chain;
4623 ChainI = 0;
4624 }
4625
4626 // TODO: MachinePointerInfo only supports a fixed length offset.
4627 MachinePointerInfo PtrInfo =
4628 !Offsets[i].isScalable() || Offsets[i].isZero()
4629 ? MachinePointerInfo(SV, Offsets[i].getKnownMinValue())
4631
4632 SDValue A = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4633 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, PtrInfo, Alignment,
4634 MMOFlags, AAInfo, Ranges);
4635 Chains[ChainI] = L.getValue(1);
4636
4637 if (MemVTs[i] != ValueVTs[i])
4638 L = DAG.getPtrExtOrTrunc(L, dl, ValueVTs[i]);
4639
4640 Values[i] = L;
4641 }
4642
4643 if (!ConstantMemory) {
4644 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4645 ArrayRef(Chains.data(), ChainI));
4646 if (isVolatile)
4647 DAG.setRoot(Chain);
4648 else
4649 PendingLoads.push_back(Chain);
4650 }
4651
4653 DAG.getVTList(ValueVTs), Values));
4654}
4655
4656void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4658 "call visitStoreToSwiftError when backend supports swifterror");
4659
4660 SmallVector<EVT, 4> ValueVTs;
4662 const Value *SrcV = I.getOperand(0);
4664 SrcV->getType(), ValueVTs, &Offsets, 0);
4665 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4666 "expect a single EVT for swifterror");
4667
4668 SDValue Src = getValue(SrcV);
4669 // Create a virtual register, then update the virtual register.
4670 Register VReg =
4671 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4672 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4673 // Chain can be getRoot or getControlRoot.
4674 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4675 SDValue(Src.getNode(), Src.getResNo()));
4676 DAG.setRoot(CopyNode);
4677}
4678
4679void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4681 "call visitLoadFromSwiftError when backend supports swifterror");
4682
4683 assert(!I.isVolatile() &&
4684 !I.hasMetadata(LLVMContext::MD_nontemporal) &&
4685 !I.hasMetadata(LLVMContext::MD_invariant_load) &&
4686 "Support volatile, non temporal, invariant for load_from_swift_error");
4687
4688 const Value *SV = I.getOperand(0);
4689 Type *Ty = I.getType();
4690 assert(
4691 (!AA ||
4694 I.getAAMetadata()))) &&
4695 "load_from_swift_error should not be constant memory");
4696
4697 SmallVector<EVT, 4> ValueVTs;
4700 ValueVTs, &Offsets, 0);
4701 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4702 "expect a single EVT for swifterror");
4703
4704 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4706 getRoot(), getCurSDLoc(),
4707 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4708
4709 setValue(&I, L);
4710}
4711
4712void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4713 if (I.isAtomic())
4714 return visitAtomicStore(I);
4715
4716 const Value *SrcV = I.getOperand(0);
4717 const Value *PtrV = I.getOperand(1);
4718
4720 if (TLI.supportSwiftError()) {
4721 // Swifterror values can come from either a function parameter with
4722 // swifterror attribute or an alloca with swifterror attribute.
4723 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4724 if (Arg->hasSwiftErrorAttr())
4725 return visitStoreToSwiftError(I);
4726 }
4727
4728 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4729 if (Alloca->isSwiftError())
4730 return visitStoreToSwiftError(I);
4731 }
4732 }
4733
4734 SmallVector<EVT, 4> ValueVTs, MemVTs;
4737 SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4738 unsigned NumValues = ValueVTs.size();
4739 if (NumValues == 0)
4740 return;
4741
4742 // Get the lowered operands. Note that we do this after
4743 // checking if NumResults is zero, because with zero results
4744 // the operands won't have values in the map.
4745 SDValue Src = getValue(SrcV);
4746 SDValue Ptr = getValue(PtrV);
4747
4748 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
4749 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4750 SDLoc dl = getCurSDLoc();
4751 Align Alignment = I.getAlign();
4752 AAMDNodes AAInfo = I.getAAMetadata();
4753
4754 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout());
4755
4756 unsigned ChainI = 0;
4757 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
4758 // See visitLoad comments.
4759 if (ChainI == MaxParallelChains) {
4760 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4761 ArrayRef(Chains.data(), ChainI));
4762 Root = Chain;
4763 ChainI = 0;
4764 }
4765
4766 // TODO: MachinePointerInfo only supports a fixed length offset.
4767 MachinePointerInfo PtrInfo =
4768 !Offsets[i].isScalable() || Offsets[i].isZero()
4769 ? MachinePointerInfo(PtrV, Offsets[i].getKnownMinValue())
4771
4772 SDValue Add = DAG.getObjectPtrOffset(dl, Ptr, Offsets[i]);
4773 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4774 if (MemVTs[i] != ValueVTs[i])
4775 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4776 SDValue St =
4777 DAG.getStore(Root, dl, Val, Add, PtrInfo, Alignment, MMOFlags, AAInfo);
4778 Chains[ChainI] = St;
4779 }
4780
4781 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4782 ArrayRef(Chains.data(), ChainI));
4783 setValue(&I, StoreNode);
4784 DAG.setRoot(StoreNode);
4785}
4786
4787void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4788 bool IsCompressing) {
4789 SDLoc sdl = getCurSDLoc();
4790
4791 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4792 Align &Alignment) {
4793 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4794 Src0 = I.getArgOperand(0);
4795 Ptr = I.getArgOperand(1);
4796 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getAlignValue();
4797 Mask = I.getArgOperand(3);
4798 };
4799 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4800 Align &Alignment) {
4801 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4802 Src0 = I.getArgOperand(0);
4803 Ptr = I.getArgOperand(1);
4804 Mask = I.getArgOperand(2);
4805 Alignment = I.getParamAlign(1).valueOrOne();
4806 };
4807
4808 Value *PtrOperand, *MaskOperand, *Src0Operand;
4809 Align Alignment;
4810 if (IsCompressing)
4811 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4812 else
4813 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4814
4815 SDValue Ptr = getValue(PtrOperand);
4816 SDValue Src0 = getValue(Src0Operand);
4817 SDValue Mask = getValue(MaskOperand);
4818 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4819
4820 EVT VT = Src0.getValueType();
4821
4822 auto MMOFlags = MachineMemOperand::MOStore;
4823 if (I.hasMetadata(LLVMContext::MD_nontemporal))
4825
4827 MachinePointerInfo(PtrOperand), MMOFlags,
4828 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4829
4830 const auto &TLI = DAG.getTargetLoweringInfo();
4831 const auto &TTI =
4832 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
4833 SDValue StoreNode =
4834 !IsCompressing &&
4835 TTI.hasConditionalLoadStoreForType(I.getArgOperand(0)->getType())
4836 ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0,
4837 Mask)
4838 : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask,
4839 VT, MMO, ISD::UNINDEXED, /*Truncating=*/false,
4840 IsCompressing);
4841 DAG.setRoot(StoreNode);
4842 setValue(&I, StoreNode);
4843}
4844
4845// Get a uniform base for the Gather/Scatter intrinsic.
4846// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4847// We try to represent it as a base pointer + vector of indices.
4848// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4849// The first operand of the GEP may be a single pointer or a vector of pointers
4850// Example:
4851// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4852// or
4853// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
4854// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4855//
4856// When the first GEP operand is a single pointer - it is the uniform base we
4857// are looking for. If first operand of the GEP is a splat vector - we
4858// extract the splat value and use it as a uniform base.
4859// In all other cases the function returns 'false'.
4860static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
4861 ISD::MemIndexType &IndexType, SDValue &Scale,
4862 SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
4863 uint64_t ElemSize) {
4864 SelectionDAG& DAG = SDB->DAG;
4865 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4866 const DataLayout &DL = DAG.getDataLayout();
4867
4868 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type");
4869
4870 // Handle splat constant pointer.
4871 if (auto *C = dyn_cast<Constant>(Ptr)) {
4872 C = C->getSplatValue();
4873 if (!C)
4874 return false;
4875
4876 Base = SDB->getValue(C);
4877
4878 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
4879 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
4880 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
4881 IndexType = ISD::SIGNED_SCALED;
4882 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4883 return true;
4884 }
4885
4886 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4887 if (!GEP || GEP->getParent() != CurBB)
4888 return false;
4889
4890 if (GEP->getNumOperands() != 2)
4891 return false;
4892
4893 const Value *BasePtr = GEP->getPointerOperand();
4894 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1);
4895
4896 // Make sure the base is scalar and the index is a vector.
4897 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy())
4898 return false;
4899
4900 TypeSize ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType());
4901 if (ScaleVal.isScalable())
4902 return false;
4903
4904 // Target may not support the required addressing mode.
4905 if (ScaleVal != 1 &&
4906 !TLI.isLegalScaleForGatherScatter(ScaleVal.getFixedValue(), ElemSize))
4907 return false;
4908
4909 Base = SDB->getValue(BasePtr);
4910 Index = SDB->getValue(IndexVal);
4911 IndexType = ISD::SIGNED_SCALED;
4912
4913 Scale =
4914 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4915 return true;
4916}
4917
4918void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4919 SDLoc sdl = getCurSDLoc();
4920
4921 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask)
4922 const Value *Ptr = I.getArgOperand(1);
4923 SDValue Src0 = getValue(I.getArgOperand(0));
4924 SDValue Mask = getValue(I.getArgOperand(3));
4925 EVT VT = Src0.getValueType();
4926 Align Alignment = cast<ConstantInt>(I.getArgOperand(2))
4927 ->getMaybeAlignValue()
4928 .value_or(DAG.getEVTAlign(VT.getScalarType()));
4930
4931 SDValue Base;
4932 SDValue Index;
4933 ISD::MemIndexType IndexType;
4934 SDValue Scale;
4935 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
4936 I.getParent(), VT.getScalarStoreSize());
4937
4938 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
4941 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata());
4942 if (!UniformBase) {
4944 Index = getValue(Ptr);
4945 IndexType = ISD::SIGNED_SCALED;
4946 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4947 }
4948
4949 EVT IdxVT = Index.getValueType();
4950 EVT EltTy = IdxVT.getVectorElementType();
4951 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
4952 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
4953 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
4954 }
4955
4956 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
4957 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4958 Ops, MMO, IndexType, false);
4959 DAG.setRoot(Scatter);
4960 setValue(&I, Scatter);
4961}
4962
4963void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4964 SDLoc sdl = getCurSDLoc();
4965
4966 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4967 Align &Alignment) {
4968 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4969 Ptr = I.getArgOperand(0);
4970 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getAlignValue();
4971 Mask = I.getArgOperand(2);
4972 Src0 = I.getArgOperand(3);
4973 };
4974 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0,
4975 Align &Alignment) {
4976 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4977 Ptr = I.getArgOperand(0);
4978 Alignment = I.getParamAlign(0).valueOrOne();
4979 Mask = I.getArgOperand(1);
4980 Src0 = I.getArgOperand(2);
4981 };
4982
4983 Value *PtrOperand, *MaskOperand, *Src0Operand;
4984 Align Alignment;
4985 if (IsExpanding)
4986 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4987 else
4988 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4989
4990 SDValue Ptr = getValue(PtrOperand);
4991 SDValue Src0 = getValue(Src0Operand);
4992 SDValue Mask = getValue(MaskOperand);
4993 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
4994
4995 EVT VT = Src0.getValueType();
4996 AAMDNodes AAInfo = I.getAAMetadata();
4997 const MDNode *Ranges = getRangeMetadata(I);
4998
4999 // Do not serialize masked loads of constant memory with anything.
5000 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
5001 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
5002
5003 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
5004
5005 auto MMOFlags = MachineMemOperand::MOLoad;
5006 if (I.hasMetadata(LLVMContext::MD_nontemporal))
5008
5010 MachinePointerInfo(PtrOperand), MMOFlags,
5011 LocationSize::beforeOrAfterPointer(), Alignment, AAInfo, Ranges);
5012
5013 const auto &TLI = DAG.getTargetLoweringInfo();
5014 const auto &TTI =
5015 TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction());
5016 // The Load/Res may point to different values and both of them are output
5017 // variables.
5018 SDValue Load;
5019 SDValue Res;
5020 if (!IsExpanding &&
5022 Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask);
5023 else
5024 Res = Load =
5025 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO,
5026 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding);
5027 if (AddToChain)
5028 PendingLoads.push_back(Load.getValue(1));
5029 setValue(&I, Res);
5030}
5031
5032void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
5033 SDLoc sdl = getCurSDLoc();
5034
5035 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
5036 const Value *Ptr = I.getArgOperand(0);
5037 SDValue Src0 = getValue(I.getArgOperand(3));
5038 SDValue Mask = getValue(I.getArgOperand(2));
5039
5041 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5042 Align Alignment = cast<ConstantInt>(I.getArgOperand(1))
5043 ->getMaybeAlignValue()
5044 .value_or(DAG.getEVTAlign(VT.getScalarType()));
5045
5046 const MDNode *Ranges = getRangeMetadata(I);
5047
5048 SDValue Root = DAG.getRoot();
5049 SDValue Base;
5050 SDValue Index;
5051 ISD::MemIndexType IndexType;
5052 SDValue Scale;
5053 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
5054 I.getParent(), VT.getScalarStoreSize());
5055 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
5058 LocationSize::beforeOrAfterPointer(), Alignment, I.getAAMetadata(),
5059 Ranges);
5060
5061 if (!UniformBase) {
5063 Index = getValue(Ptr);
5064 IndexType = ISD::SIGNED_SCALED;
5065 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
5066 }
5067
5068 EVT IdxVT = Index.getValueType();
5069 EVT EltTy = IdxVT.getVectorElementType();
5070 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
5071 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
5072 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
5073 }
5074
5075 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
5076 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
5077 Ops, MMO, IndexType, ISD::NON_EXTLOAD);
5078
5079 PendingLoads.push_back(Gather.getValue(1));
5080 setValue(&I, Gather);
5081}
5082
5083void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
5084 SDLoc dl = getCurSDLoc();
5085 AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
5086 AtomicOrdering FailureOrdering = I.getFailureOrdering();
5087 SyncScope::ID SSID = I.getSyncScopeID();
5088
5089 SDValue InChain = getRoot();
5090
5091 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
5092 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
5093
5096
5099 MachinePointerInfo(I.getPointerOperand()), Flags,
5101 AAMDNodes(), nullptr, SSID, SuccessOrdering, FailureOrdering);
5102
5104 dl, MemVT, VTs, InChain,
5105 getValue(I.getPointerOperand()),
5106 getValue(I.getCompareOperand()),
5107 getValue(I.getNewValOperand()), MMO);
5108
5109 SDValue OutChain = L.getValue(2);
5110
5111 setValue(&I, L);
5112 DAG.setRoot(OutChain);
5113}
5114
5115void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
5116 SDLoc dl = getCurSDLoc();
5118 switch (I.getOperation()) {
5119 default: llvm_unreachable("Unknown atomicrmw operation");
5137 break;
5140 break;
5143 break;
5146 break;
5147 }
5148 AtomicOrdering Ordering = I.getOrdering();
5149 SyncScope::ID SSID = I.getSyncScopeID();
5150
5151 SDValue InChain = getRoot();
5152
5153 auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
5156
5159 MachinePointerInfo(I.getPointerOperand()), Flags,
5161 AAMDNodes(), nullptr, SSID, Ordering);
5162
5163 SDValue L =
5164 DAG.getAtomic(NT, dl, MemVT, InChain,
5165 getValue(I.getPointerOperand()), getValue(I.getValOperand()),
5166 MMO);
5167
5168 SDValue OutChain = L.getValue(1);
5169
5170 setValue(&I, L);
5171 DAG.setRoot(OutChain);
5172}
5173
5174void SelectionDAGBuilder::visitFence(const FenceInst &I) {
5175 SDLoc dl = getCurSDLoc();
5177 SDValue Ops[3];
5178 Ops[0] = getRoot();
5179 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl,
5181 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl,
5183 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops);
5184 setValue(&I, N);
5185 DAG.setRoot(N);
5186}
5187
5188void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
5189 SDLoc dl = getCurSDLoc();
5190 AtomicOrdering Order = I.getOrdering();
5191 SyncScope::ID SSID = I.getSyncScopeID();
5192
5193 SDValue InChain = getRoot();
5194
5196 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5197 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
5198
5199 if (!TLI.supportsUnalignedAtomics() &&
5200 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5201 report_fatal_error("Cannot generate unaligned atomic load");
5202
5204
5206 MachinePointerInfo(I.getPointerOperand()), Flags,
5207 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5208 nullptr, SSID, Order);
5209
5210 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
5211
5212 SDValue Ptr = getValue(I.getPointerOperand());
5213 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
5214 Ptr, MMO);
5215
5216 SDValue OutChain = L.getValue(1);
5217 if (MemVT != VT)
5218 L = DAG.getPtrExtOrTrunc(L, dl, VT);
5219
5220 setValue(&I, L);
5221 DAG.setRoot(OutChain);
5222}
5223
5224void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
5225 SDLoc dl = getCurSDLoc();
5226
5227 AtomicOrdering Ordering = I.getOrdering();
5228 SyncScope::ID SSID = I.getSyncScopeID();
5229
5230 SDValue InChain = getRoot();
5231
5233 EVT MemVT =
5234 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
5235
5236 if (!TLI.supportsUnalignedAtomics() &&
5237 I.getAlign().value() < MemVT.getSizeInBits() / 8)
5238 report_fatal_error("Cannot generate unaligned atomic store");
5239
5241
5244 MachinePointerInfo(I.getPointerOperand()), Flags,
5245 LocationSize::precise(MemVT.getStoreSize()), I.getAlign(), AAMDNodes(),
5246 nullptr, SSID, Ordering);
5247
5248 SDValue Val = getValue(I.getValueOperand());
5249 if (Val.getValueType() != MemVT)
5250 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
5251 SDValue Ptr = getValue(I.getPointerOperand());
5252
5253 SDValue OutChain =
5254 DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
5255
5256 setValue(&I, OutChain);
5257 DAG.setRoot(OutChain);
5258}
5259
5260/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
5261/// node.
5262void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
5263 unsigned Intrinsic) {
5264 // Ignore the callsite's attributes. A specific call site may be marked with
5265 // readnone, but the lowering code will expect the chain based on the
5266 // definition.
5267 const Function *F = I.getCalledFunction();
5268 bool HasChain = !F->doesNotAccessMemory();
5269 bool OnlyLoad =
5270 HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
5271
5272 // Build the operand list.
5274 if (HasChain) { // If this intrinsic has side-effects, chainify it.
5275 if (OnlyLoad) {
5276 // We don't need to serialize loads against other loads.
5277 Ops.push_back(DAG.getRoot());
5278 } else {
5279 Ops.push_back(getRoot());
5280 }
5281 }
5282
5283 // Info is set by getTgtMemIntrinsic
5286 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
5288 Intrinsic);
5289
5290 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
5291 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
5293 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
5295
5296 // Add all operands of the call to the operand list.
5297 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) {
5298 const Value *Arg = I.getArgOperand(i);
5299 if (!I.paramHasAttr(i, Attribute::ImmArg)) {
5300 Ops.push_back(getValue(Arg));
5301 continue;
5302 }
5303
5304 // Use TargetConstant instead of a regular constant for immarg.
5305 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true);
5306 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
5307 assert(CI->getBitWidth() <= 64 &&
5308 "large intrinsic immediates not handled");
5309 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
5310 } else {
5311 Ops.push_back(
5312 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
5313 }
5314 }
5315
5316 SmallVector<EVT, 4> ValueVTs;
5317 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
5318
5319 if (HasChain)
5320 ValueVTs.push_back(MVT::Other);
5321
5322 SDVTList VTs = DAG.getVTList(ValueVTs);
5323
5324 // Propagate fast-math-flags from IR to node(s).
5326 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
5327 Flags.copyFMF(*FPMO);
5328 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
5329
5330 // Create the node.
5332
5333 if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
5334 auto *Token = Bundle->Inputs[0].get();
5335 SDValue ConvControlToken = getValue(Token);
5336 assert(Ops.back().getValueType() != MVT::Glue &&
5337 "Did not expected another glue node here.");
5338 ConvControlToken =
5339 DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken);
5340 Ops.push_back(ConvControlToken);
5341 }
5342
5343 // In some cases, custom collection of operands from CallInst I may be needed.
5345 if (IsTgtIntrinsic) {
5346 // This is target intrinsic that touches memory
5347 //
5348 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
5349 // didn't yield anything useful.
5351 if (Info.ptrVal)
5352 MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
5353 else if (Info.fallbackAddressSpace)
5354 MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
5355 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops,
5356 Info.memVT, MPI, Info.align, Info.flags,
5357 Info.size, I.getAAMetadata());
5358 } else if (!HasChain) {
5360 } else if (!I.getType()->isVoidTy()) {
5362 } else {
5364 }
5365
5366 if (HasChain) {
5367 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
5368 if (OnlyLoad)
5369 PendingLoads.push_back(Chain);
5370 else
5371 DAG.setRoot(Chain);
5372 }
5373
5374 if (!I.getType()->isVoidTy()) {
5375 if (!isa<VectorType>(I.getType()))
5376 Result = lowerRangeToAssertZExt(DAG, I, Result);
5377
5378 MaybeAlign Alignment = I.getRetAlign();
5379
5380 // Insert `assertalign` node if there's an alignment.
5381 if (InsertAssertAlign && Alignment) {
5382 Result =
5383 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne());
5384 }
5385 }
5386
5387 setValue(&I, Result);
5388}
5389
5390/// GetSignificand - Get the significand and build it into a floating-point
5391/// number with exponent of 1:
5392///
5393/// Op = (Op & 0x007fffff) | 0x3f800000;
5394///
5395/// where Op is the hexadecimal representation of floating point value.
5397 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5398 DAG.getConstant(0x007fffff, dl, MVT::i32));
5399 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
5400 DAG.getConstant(0x3f800000, dl, MVT::i32));
5401 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
5402}
5403
5404/// GetExponent - Get the exponent:
5405///
5406/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
5407///
5408/// where Op is the hexadecimal representation of floating point value.
5410 const TargetLowering &TLI, const SDLoc &dl) {
5411 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
5412 DAG.getConstant(0x7f800000, dl, MVT::i32));
5413 SDValue t1 = DAG.getNode(
5414 ISD::SRL, dl, MVT::i32, t0,
5415 DAG.getConstant(23, dl,
5416 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
5417 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
5418 DAG.getConstant(127, dl, MVT::i32));
5419 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
5420}
5421
5422/// getF32Constant - Get 32-bit floating point constant.
5424 const SDLoc &dl) {
5425 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
5426 MVT::f32);
5427}
5428
5430 SelectionDAG &DAG) {
5431 // TODO: What fast-math-flags should be set on the floating-point nodes?
5432
5433 // IntegerPartOfX = ((int32_t)(t0);
5434 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
5435
5436 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
5437 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
5438 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
5439
5440 // IntegerPartOfX <<= 23;
5441 IntegerPartOfX =
5442 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX,
5443 DAG.getConstant(23, dl,
5445 MVT::i32, DAG.getDataLayout())));
5446
5447 SDValue TwoToFractionalPartOfX;
5448 if (LimitFloatPrecision <= 6) {
5449 // For floating-point precision of 6:
5450 //
5451 // TwoToFractionalPartOfX =
5452 // 0.997535578f +
5453 // (0.735607626f + 0.252464424f * x) * x;
5454 //
5455 // error 0.0144103317, which is 6 bits
5456 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5457 getF32Constant(DAG, 0x3e814304, dl));
5458 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5459 getF32Constant(DAG, 0x3f3c50c8, dl));
5460 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5461 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5462 getF32Constant(DAG, 0x3f7f5e7e, dl));
5463 } else if (LimitFloatPrecision <= 12) {
5464 // For floating-point precision of 12:
5465 //
5466 // TwoToFractionalPartOfX =
5467 // 0.999892986f +
5468 // (0.696457318f +
5469 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
5470 //
5471 // error 0.000107046256, which is 13 to 14 bits
5472 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5473 getF32Constant(DAG, 0x3da235e3, dl));
5474 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5475 getF32Constant(DAG, 0x3e65b8f3, dl));
5476 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5477 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5478 getF32Constant(DAG, 0x3f324b07, dl));
5479 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5480 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5481 getF32Constant(DAG, 0x3f7ff8fd, dl));
5482 } else { // LimitFloatPrecision <= 18
5483 // For floating-point precision of 18:
5484 //
5485 // TwoToFractionalPartOfX =
5486 // 0.999999982f +
5487 // (0.693148872f +
5488 // (0.240227044f +
5489 // (0.554906021e-1f +
5490 // (0.961591928e-2f +
5491 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
5492 // error 2.47208000*10^(-7), which is better than 18 bits
5493 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5494 getF32Constant(DAG, 0x3924b03e, dl));
5495 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5496 getF32Constant(DAG, 0x3ab24b87, dl));
5497 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5498 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5499 getF32Constant(DAG, 0x3c1d8c17, dl));
5500 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5501 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5502 getF32Constant(DAG, 0x3d634a1d, dl));
5503 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5504 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5505 getF32Constant(DAG, 0x3e75fe14, dl));
5506 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5507 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
5508 getF32Constant(DAG, 0x3f317234, dl));
5509 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
5510 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
5511 getF32Constant(DAG, 0x3f800000, dl));
5512 }
5513
5514 // Add the exponent into the result in integer domain.
5515 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
5516 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5517 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
5518}
5519
5520/// expandExp - Lower an exp intrinsic. Handles the special sequences for
5521/// limited-precision mode.
5523 const TargetLowering &TLI, SDNodeFlags Flags) {
5524 if (Op.getValueType() == MVT::f32 &&
5526
5527 // Put the exponent in the right bit position for later addition to the
5528 // final result:
5529 //
5530 // t0 = Op * log2(e)
5531
5532 // TODO: What fast-math-flags should be set here?
5533 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
5534 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32));
5535 return getLimitedPrecisionExp2(t0, dl, DAG);
5536 }
5537
5538 // No special expansion.
5539 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags);
5540}
5541
5542/// expandLog - Lower a log intrinsic. Handles the special sequences for
5543/// limited-precision mode.
5545 const TargetLowering &TLI, SDNodeFlags Flags) {
5546 // TODO: What fast-math-flags should be set on the floating-point nodes?
5547
5548 if (Op.getValueType() == MVT::f32 &&
5550 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5551
5552 // Scale the exponent by log(2).
5553 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5554 SDValue LogOfExponent =
5555 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5556 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32));
5557
5558 // Get the significand and build it into a floating-point number with
5559 // exponent of 1.
5560 SDValue X = GetSignificand(DAG, Op1, dl);
5561
5562 SDValue LogOfMantissa;
5563 if (LimitFloatPrecision <= 6) {
5564 // For floating-point precision of 6:
5565 //
5566 // LogofMantissa =
5567 // -1.1609546f +
5568 // (1.4034025f - 0.23903021f * x) * x;
5569 //
5570 // error 0.0034276066, which is better than 8 bits
5571 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5572 getF32Constant(DAG, 0xbe74c456, dl));
5573 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5574 getF32Constant(DAG, 0x3fb3a2b1, dl));
5575 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5576 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5577 getF32Constant(DAG, 0x3f949a29, dl));
5578 } else if (LimitFloatPrecision <= 12) {
5579 // For floating-point precision of 12:
5580 //
5581 // LogOfMantissa =
5582 // -1.7417939f +
5583 // (2.8212026f +
5584 // (-1.4699568f +
5585 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
5586 //
5587 // error 0.000061011436, which is 14 bits
5588 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5589 getF32Constant(DAG, 0xbd67b6d6, dl));
5590 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5591 getF32Constant(DAG, 0x3ee4f4b8, dl));
5592 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5593 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5594 getF32Constant(DAG, 0x3fbc278b, dl));
5595 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5596 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5597 getF32Constant(DAG, 0x40348e95, dl));
5598 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5599 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5600 getF32Constant(DAG, 0x3fdef31a, dl));
5601 } else { // LimitFloatPrecision <= 18
5602 // For floating-point precision of 18:
5603 //
5604 // LogOfMantissa =
5605 // -2.1072184f +
5606 // (4.2372794f +
5607 // (-3.7029485f +
5608 // (2.2781945f +
5609 // (-0.87823314f +
5610 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5611 //
5612 // error 0.0000023660568, which is better than 18 bits
5613 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5614 getF32Constant(DAG, 0xbc91e5ac, dl));
5615 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5616 getF32Constant(DAG, 0x3e4350aa, dl));
5617 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5618 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5619 getF32Constant(DAG, 0x3f60d3e3, dl));
5620 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5621 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5622 getF32Constant(DAG, 0x4011cdf0, dl));
5623 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5624 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5625 getF32Constant(DAG, 0x406cfd1c, dl));
5626 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5627 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5628 getF32Constant(DAG, 0x408797cb, dl));
5629 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5630 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5631 getF32Constant(DAG, 0x4006dcab, dl));
5632 }
5633
5634 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5635 }
5636
5637 // No special expansion.
5638 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags);
5639}
5640
5641/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5642/// limited-precision mode.
5644 const TargetLowering &TLI, SDNodeFlags Flags) {
5645 // TODO: What fast-math-flags should be set on the floating-point nodes?
5646
5647 if (Op.getValueType() == MVT::f32 &&
5649 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5650
5651 // Get the exponent.
5652 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5653
5654 // Get the significand and build it into a floating-point number with
5655 // exponent of 1.
5656 SDValue X = GetSignificand(DAG, Op1, dl);
5657
5658 // Different possible minimax approximations of significand in
5659 // floating-point for various degrees of accuracy over [1,2].
5660 SDValue Log2ofMantissa;
5661 if (LimitFloatPrecision <= 6) {
5662 // For floating-point precision of 6:
5663 //
5664 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
5665 //
5666 // error 0.0049451742, which is more than 7 bits
5667 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5668 getF32Constant(DAG, 0xbeb08fe0, dl));
5669 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5670 getF32Constant(DAG, 0x40019463, dl));
5671 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5672 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5673 getF32Constant(DAG, 0x3fd6633d, dl));
5674 } else if (LimitFloatPrecision <= 12) {
5675 // For floating-point precision of 12:
5676 //
5677 // Log2ofMantissa =
5678 // -2.51285454f +
5679 // (4.07009056f +
5680 // (-2.12067489f +
5681 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
5682 //
5683 // error 0.0000876136000, which is better than 13 bits
5684 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5685 getF32Constant(DAG, 0xbda7262e, dl));
5686 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5687 getF32Constant(DAG, 0x3f25280b, dl));
5688 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5689 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5690 getF32Constant(DAG, 0x4007b923, dl));
5691 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5692 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5693 getF32Constant(DAG, 0x40823e2f, dl));
5694 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5695 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5696 getF32Constant(DAG, 0x4020d29c, dl));
5697 } else { // LimitFloatPrecision <= 18
5698 // For floating-point precision of 18:
5699 //
5700 // Log2ofMantissa =
5701 // -3.0400495f +
5702 // (6.1129976f +
5703 // (-5.3420409f +
5704 // (3.2865683f +
5705 // (-1.2669343f +
5706 // (0.27515199f -
5707 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
5708 //
5709 // error 0.0000018516, which is better than 18 bits
5710 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5711 getF32Constant(DAG, 0xbcd2769e, dl));
5712 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5713 getF32Constant(DAG, 0x3e8ce0b9, dl));
5714 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5715 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5716 getF32Constant(DAG, 0x3fa22ae7, dl));
5717 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5718 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5719 getF32Constant(DAG, 0x40525723, dl));
5720 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5721 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5722 getF32Constant(DAG, 0x40aaf200, dl));
5723 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5724 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5725 getF32Constant(DAG, 0x40c39dad, dl));
5726 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5727 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5728 getF32Constant(DAG, 0x4042902c, dl));
5729 }
5730
5731 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
5732 }
5733
5734 // No special expansion.
5735 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags);
5736}
5737
5738/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
5739/// limited-precision mode.
5741 const TargetLowering &TLI, SDNodeFlags Flags) {
5742 // TODO: What fast-math-flags should be set on the floating-point nodes?
5743
5744 if (Op.getValueType() == MVT::f32 &&
5746 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5747
5748 // Scale the exponent by log10(2) [0.30102999f].
5749 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
5750 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
5751 getF32Constant(DAG, 0x3e9a209a, dl));
5752
5753 // Get the significand and build it into a floating-point number with
5754 // exponent of 1.
5755 SDValue X = GetSignificand(DAG, Op1, dl);
5756
5757 SDValue Log10ofMantissa;
5758 if (LimitFloatPrecision <= 6) {
5759 // For floating-point precision of 6:
5760 //
5761 // Log10ofMantissa =
5762 // -0.50419619f +
5763 // (0.60948995f - 0.10380950f * x) * x;
5764 //
5765 // error 0.0014886165, which is 6 bits
5766 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5767 getF32Constant(DAG, 0xbdd49a13, dl));
5768 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5769 getF32Constant(DAG, 0x3f1c0789, dl));
5770 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5771 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5772 getF32Constant(DAG, 0x3f011300, dl));
5773 } else if (LimitFloatPrecision <= 12) {
5774 // For floating-point precision of 12:
5775 //
5776 // Log10ofMantissa =
5777 // -0.64831180f +
5778 // (0.91751397f +
5779 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
5780 //
5781 // error 0.00019228036, which is better than 12 bits
5782 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5783 getF32Constant(DAG, 0x3d431f31, dl));
5784 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5785 getF32Constant(DAG, 0x3ea21fb2, dl));
5786 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5787 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5788 getF32Constant(DAG, 0x3f6ae232, dl));
5789 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5790 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5791 getF32Constant(DAG, 0x3f25f7c3, dl));
5792 } else { // LimitFloatPrecision <= 18
5793 // For floating-point precision of 18:
5794 //
5795 // Log10ofMantissa =
5796 // -0.84299375f +
5797 // (1.5327582f +
5798 // (-1.0688956f +
5799 // (0.49102474f +
5800 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
5801 //
5802 // error 0.0000037995730, which is better than 18 bits
5803 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5804 getF32Constant(DAG, 0x3c5d51ce, dl));
5805 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
5806 getF32Constant(DAG, 0x3e00685a, dl));
5807 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5808 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
5809 getF32Constant(DAG, 0x3efb6798, dl));
5810 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5811 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
5812 getF32Constant(DAG, 0x3f88d192, dl));
5813 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5814 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
5815 getF32Constant(DAG, 0x3fc4316c, dl));
5816 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5817 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
5818 getF32Constant(DAG, 0x3f57ce70, dl));
5819 }
5820
5821 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
5822 }
5823
5824 // No special expansion.
5825 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags);
5826}
5827
5828/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
5829/// limited-precision mode.
5831 const TargetLowering &TLI, SDNodeFlags Flags) {
5832 if (Op.getValueType() == MVT::f32 &&
5834 return getLimitedPrecisionExp2(Op, dl, DAG);
5835
5836 // No special expansion.
5837 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags);
5838}
5839
5840/// visitPow - Lower a pow intrinsic. Handles the special sequences for
5841/// limited-precision mode with x == 10.0f.
5842static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
5843 SelectionDAG &DAG, const TargetLowering &TLI,
5844 SDNodeFlags Flags) {
5845 bool IsExp10 = false;
5846 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
5848 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
5849 APFloat Ten(10.0f);
5850 IsExp10 = LHSC->isExactlyValue(Ten);
5851 }
5852 }
5853
5854 // TODO: What fast-math-flags should be set on the FMUL node?
5855 if (IsExp10) {
5856 // Put the exponent in the right bit position for later addition to the
5857 // final result:
5858 //
5859 // #define LOG2OF10 3.3219281f
5860 // t0 = Op * LOG2OF10;
5861 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
5862 getF32Constant(DAG, 0x40549a78, dl));
5863 return getLimitedPrecisionExp2(t0, dl, DAG);
5864 }
5865
5866 // No special expansion.
5867 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags);
5868}
5869
5870/// ExpandPowI - Expand a llvm.powi intrinsic.
5871static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
5872 SelectionDAG &DAG) {
5873 // If RHS is a constant, we can expand this out to a multiplication tree if
5874 // it's beneficial on the target, otherwise we end up lowering to a call to
5875 // __powidf2 (for example).
5876 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
5877 unsigned Val = RHSC->getSExtValue();
5878
5879 // powi(x, 0) -> 1.0
5880 if (Val == 0)
5881 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
5882
5884 Val, DAG.shouldOptForSize())) {
5885 // Get the exponent as a positive value.
5886 if ((int)Val < 0)
5887 Val = -Val;
5888 // We use the simple binary decomposition method to generate the multiply
5889 // sequence. There are more optimal ways to do this (for example,
5890 // powi(x,15) generates one more multiply than it should), but this has
5891 // the benefit of being both really simple and much better than a libcall.
5892 SDValue Res; // Logically starts equal to 1.0
5893 SDValue CurSquare = LHS;
5894 // TODO: Intrinsics should have fast-math-flags that propagate to these
5895 // nodes.
5896 while (Val) {
5897 if (Val & 1) {
5898 if (Res.getNode())
5899 Res =
5900 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare);
5901 else
5902 Res = CurSquare; // 1.0*CurSquare.
5903 }
5904
5905 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
5906 CurSquare, CurSquare);
5907 Val >>= 1;
5908 }
5909
5910 // If the original was negative, invert the result, producing 1/(x*x*x).
5911 if (RHSC->getSExtValue() < 0)
5912 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
5913 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
5914 return Res;
5915 }
5916 }
5917
5918 // Otherwise, expand to a libcall.
5919 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
5920}
5921
5922static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL,
5923 SDValue LHS, SDValue RHS, SDValue Scale,
5924 SelectionDAG &DAG, const TargetLowering &TLI) {
5925 EVT VT = LHS.getValueType();
5926 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
5927 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
5928 LLVMContext &Ctx = *DAG.getContext();
5929
5930 // If the type is legal but the operation isn't, this node might survive all
5931 // the way to operation legalization. If we end up there and we do not have
5932 // the ability to widen the type (if VT*2 is not legal), we cannot expand the
5933 // node.
5934
5935 // Coax the legalizer into expanding the node during type legalization instead
5936 // by bumping the size by one bit. This will force it to Promote, enabling the
5937 // early expansion and avoiding the need to expand later.
5938
5939 // We don't have to do this if Scale is 0; that can always be expanded, unless
5940 // it's a saturating signed operation. Those can experience true integer
5941 // division overflow, a case which we must avoid.
5942
5943 // FIXME: We wouldn't have to do this (or any of the early
5944 // expansion/promotion) if it was possible to expand a libcall of an
5945 // illegal type during operation legalization. But it's not, so things
5946 // get a bit hacky.
5947 unsigned ScaleInt = Scale->getAsZExtVal();
5948 if ((ScaleInt > 0 || (Saturating && Signed)) &&
5949 (TLI.isTypeLegal(VT) ||
5950 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) {
5952 Opcode, VT, ScaleInt);
5953 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) {
5954 EVT PromVT;
5955 if (VT.isScalarInteger())
5956 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1);
5957 else if (VT.isVector()) {
5958 PromVT = VT.getVectorElementType();
5959 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1);
5960 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount());
5961 } else
5962 llvm_unreachable("Wrong VT for DIVFIX?");
5963 LHS = DAG.getExtOrTrunc(Signed, LHS, DL, PromVT);
5964 RHS = DAG.getExtOrTrunc(Signed, RHS, DL, PromVT);
5965 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout());
5966 // For saturating operations, we need to shift up the LHS to get the
5967 // proper saturation width, and then shift down again afterwards.
5968 if (Saturating)
5969 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS,
5970 DAG.getConstant(1, DL, ShiftTy));
5971 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale);
5972 if (Saturating)
5973 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res,
5974 DAG.getConstant(1, DL, ShiftTy));
5975 return DAG.getZExtOrTrunc(Res, DL, VT);
5976 }
5977 }
5978
5979 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale);
5980}
5981
5982// getUnderlyingArgRegs - Find underlying registers used for a truncated,
5983// bitcasted, or split argument. Returns a list of <Register, size in bits>
5984static void
5985getUnderlyingArgRegs(SmallVectorImpl<std::pair<Register, TypeSize>> &Regs,
5986 const SDValue &N) {
5987 switch (N.getOpcode()) {
5988 case ISD::CopyFromReg: {
5989 SDValue Op = N.getOperand(1);
5990 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(),
5991 Op.getValueType().getSizeInBits());
5992 return;
5993 }
5994 case ISD::BITCAST:
5995 case ISD::AssertZext:
5996 case ISD::AssertSext:
5997 case ISD::TRUNCATE:
5998 getUnderlyingArgRegs(Regs, N.getOperand(0));
5999 return;
6000 case ISD::BUILD_PAIR:
6001 case ISD::BUILD_VECTOR:
6003 for (SDValue Op : N->op_values())
6004 getUnderlyingArgRegs(Regs, Op);
6005 return;
6006 default:
6007 return;
6008 }
6009}
6010
6011/// If the DbgValueInst is a dbg_value of a function argument, create the
6012/// corresponding DBG_VALUE machine instruction for it now. At the end of
6013/// instruction selection, they will be inserted to the entry BB.
6014/// We don't currently support this for variadic dbg_values, as they shouldn't
6015/// appear for function arguments or in the prologue.
6016bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
6017 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
6018 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) {
6019 const Argument *Arg = dyn_cast<Argument>(V);
6020 if (!Arg)
6021 return false;
6022
6025
6026 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind
6027 // we've been asked to pursue.
6028 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr,
6029 bool Indirect) {
6030 if (Reg.isVirtual() && MF.useDebugInstrRef()) {
6031 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF
6032 // pointing at the VReg, which will be patched up later.
6033 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF);
6035 /* Reg */ Reg, /* isDef */ false, /* isImp */ false,
6036 /* isKill */ false, /* isDead */ false,
6037 /* isUndef */ false, /* isEarlyClobber */ false,
6038 /* SubReg */ 0, /* isDebug */ true)});
6039
6040 auto *NewDIExpr = FragExpr;
6041 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into
6042 // the DIExpression.
6043 if (Indirect)
6044 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore);
6046 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops);
6047 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr);
6048 } else {
6049 // Create a completely standard DBG_VALUE.
6050 auto &Inst = TII->get(TargetOpcode::DBG_VALUE);
6051 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr);
6052 }
6053 };
6054
6055 if (Kind == FuncArgumentDbgValueKind::Value) {
6056 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6057 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
6058 // the entry block.
6059 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
6060 if (!IsInEntryBlock)
6061 return false;
6062
6063 // ArgDbgValues are hoisted to the beginning of the entry block. So we
6064 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
6065 // variable that also is a param.
6066 //
6067 // Although, if we are at the top of the entry block already, we can still
6068 // emit using ArgDbgValue. This might catch some situations when the
6069 // dbg.value refers to an argument that isn't used in the entry block, so
6070 // any CopyToReg node would be optimized out and the only way to express
6071 // this DBG_VALUE is by using the physical reg (or FI) as done in this
6072 // method. ArgDbgValues are hoisted to the beginning of the entry block. So
6073 // we should only emit as ArgDbgValue if the Variable is an argument to the
6074 // current function, and the dbg.value intrinsic is found in the entry
6075 // block.
6076 bool VariableIsFunctionInputArg = Variable->isParameter() &&
6077 !DL->getInlinedAt();
6078 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
6079 if (!IsInPrologue && !VariableIsFunctionInputArg)
6080 return false;
6081
6082 // Here we assume that a function argument on IR level only can be used to
6083 // describe one input parameter on source level. If we for example have
6084 // source code like this
6085 //
6086 // struct A { long x, y; };
6087 // void foo(struct A a, long b) {
6088 // ...
6089 // b = a.x;
6090 // ...
6091 // }
6092 //
6093 // and IR like this
6094 //
6095 // define void @foo(i32 %a1, i32 %a2, i32 %b) {
6096 // entry:
6097 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
6098 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
6099 // call void @llvm.dbg.value(metadata i32 %b, "b",
6100 // ...
6101 // call void @llvm.dbg.value(metadata i32 %a1, "b"
6102 // ...
6103 //
6104 // then the last dbg.value is describing a parameter "b" using a value that
6105 // is an argument. But since we already has used %a1 to describe a parameter
6106 // we should not handle that last dbg.value here (that would result in an
6107 // incorrect hoisting of the DBG_VALUE to the function entry).
6108 // Notice that we allow one dbg.value per IR level argument, to accommodate
6109 // for the situation with fragments above.
6110 // If there is no node for the value being handled, we return true to skip
6111 // the normal generation of debug info, as it would kill existing debug
6112 // info for the parameter in case of duplicates.
6113 if (VariableIsFunctionInputArg) {
6114 unsigned ArgNo = Arg->getArgNo();
6115 if (ArgNo >= FuncInfo.DescribedArgs.size())
6116 FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
6117 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
6118 return !NodeMap[V].getNode();
6119 FuncInfo.DescribedArgs.set(ArgNo);
6120 }
6121 }
6122
6123 bool IsIndirect = false;
6124 std::optional<MachineOperand> Op;
6125 // Some arguments' frame index is recorded during argument lowering.
6126 int FI = FuncInfo.getArgumentFrameIndex(Arg);
6127 if (FI != std::numeric_limits<int>::max())
6129
6131 if (!Op && N.getNode()) {
6132 getUnderlyingArgRegs(ArgRegsAndSizes, N);
6133 Register Reg;
6134 if (ArgRegsAndSizes.size() == 1)
6135 Reg = ArgRegsAndSizes.front().first;
6136
6137 if (Reg && Reg.isVirtual()) {
6139 Register PR = RegInfo.getLiveInPhysReg(Reg);
6140 if (PR)
6141 Reg = PR;
6142 }
6143 if (Reg) {
6144 Op = MachineOperand::CreateReg(Reg, false);
6145 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6146 }
6147 }
6148
6149 if (!Op && N.getNode()) {
6150 // Check if frame index is available.
6151 SDValue LCandidate = peekThroughBitcasts(N);
6152 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode()))
6153 if (FrameIndexSDNode *FINode =
6154 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
6155 Op = MachineOperand::CreateFI(FINode->getIndex());
6156 }
6157
6158 if (!Op) {
6159 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg
6160 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<Register, TypeSize>>
6161 SplitRegs) {
6162 unsigned Offset = 0;
6163 for (const auto &RegAndSize : SplitRegs) {
6164 // If the expression is already a fragment, the current register
6165 // offset+size might extend beyond the fragment. In this case, only
6166 // the register bits that are inside the fragment are relevant.
6167 int RegFragmentSizeInBits = RegAndSize.second;
6168 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) {
6169 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits;
6170 // The register is entirely outside the expression fragment,
6171 // so is irrelevant for debug info.
6172 if (Offset >= ExprFragmentSizeInBits)
6173 break;
6174 // The register is partially outside the expression fragment, only
6175 // the low bits within the fragment are relevant for debug info.
6176 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) {
6177 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset;
6178 }
6179 }
6180
6181 auto FragmentExpr = DIExpression::createFragmentExpression(
6182 Expr, Offset, RegFragmentSizeInBits);
6183 Offset += RegAndSize.second;
6184 // If a valid fragment expression cannot be created, the variable's
6185 // correct value cannot be determined and so it is set as Undef.
6186 if (!FragmentExpr) {
6188 Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder);
6189 DAG.AddDbgValue(SDV, false);
6190 continue;
6191 }
6192 MachineInstr *NewMI =
6193 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr,
6194 Kind != FuncArgumentDbgValueKind::Value);
6195 FuncInfo.ArgDbgValues.push_back(NewMI);
6196 }
6197 };
6198
6199 // Check if ValueMap has reg number.
6201 VMI = FuncInfo.ValueMap.find(V);
6202 if (VMI != FuncInfo.ValueMap.end()) {
6203 const auto &TLI = DAG.getTargetLoweringInfo();
6204 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
6205 V->getType(), std::nullopt);
6206 if (RFV.occupiesMultipleRegs()) {
6207 splitMultiRegDbgValue(RFV.getRegsAndSizes());
6208 return true;
6209 }
6210
6211 Op = MachineOperand::CreateReg(VMI->second, false);
6212 IsIndirect = Kind != FuncArgumentDbgValueKind::Value;
6213 } else if (ArgRegsAndSizes.size() > 1) {
6214 // This was split due to the calling convention, and no virtual register
6215 // mapping exists for the value.
6216 splitMultiRegDbgValue(ArgRegsAndSizes);
6217 return true;
6218 }
6219 }
6220
6221 if (!Op)
6222 return false;
6223
6225 "Expected inlined-at fields to agree");
6226 MachineInstr *NewMI = nullptr;
6227
6228 if (Op->isReg())
6229 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect);
6230 else
6231 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op,
6232 Variable, Expr);
6233
6234 // Otherwise, use ArgDbgValues.
6235 FuncInfo.ArgDbgValues.push_back(NewMI);
6236 return true;
6237}
6238
6239/// Return the appropriate SDDbgValue based on N.
6240SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
6241 DILocalVariable *Variable,
6242 DIExpression *Expr,
6243 const DebugLoc &dl,
6244 unsigned DbgSDNodeOrder) {
6245 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
6246 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
6247 // stack slot locations.
6248 //
6249 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
6250 // debug values here after optimization:
6251 //
6252 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
6253 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
6254 //
6255 // Both describe the direct values of their associated variables.
6256 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
6257 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6258 }
6259 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
6260 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
6261}
6262
6263static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
6264 switch (Intrinsic) {
6265 case Intrinsic::smul_fix:
6266 return ISD::SMULFIX;
6267 case Intrinsic::umul_fix:
6268 return ISD::UMULFIX;
6269 case Intrinsic::smul_fix_sat:
6270 return ISD::SMULFIXSAT;
6271 case Intrinsic::umul_fix_sat:
6272 return ISD::UMULFIXSAT;
6273 case Intrinsic::sdiv_fix:
6274 return ISD::SDIVFIX;
6275 case Intrinsic::udiv_fix:
6276 return ISD::UDIVFIX;
6277 case Intrinsic::sdiv_fix_sat:
6278 return ISD::SDIVFIXSAT;
6279 case Intrinsic::udiv_fix_sat:
6280 return ISD::UDIVFIXSAT;
6281 default:
6282 llvm_unreachable("Unhandled fixed point intrinsic");
6283 }
6284}
6285
6286void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I,
6287 const char *FunctionName) {
6288 assert(FunctionName && "FunctionName must not be nullptr");
6290 FunctionName,
6292 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
6293}
6294
6295/// Given a @llvm.call.preallocated.setup, return the corresponding
6296/// preallocated call.
6297static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) {
6298 assert(cast<CallBase>(PreallocatedSetup)
6300 ->getIntrinsicID() == Intrinsic::call_preallocated_setup &&
6301 "expected call_preallocated_setup Value");
6302 for (const auto *U : PreallocatedSetup->users()) {
6303 auto *UseCall = cast<CallBase>(U);
6304 const Function *Fn = UseCall->getCalledFunction();
6305 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) {
6306 return UseCall;
6307 }
6308 }
6309 llvm_unreachable("expected corresponding call to preallocated setup/arg");
6310}
6311
6312/// If DI is a debug value with an EntryValue expression, lower it using the
6313/// corresponding physical register of the associated Argument value
6314/// (guaranteed to exist by the verifier).
6315bool SelectionDAGBuilder::visitEntryValueDbgValue(
6316 ArrayRef<const Value *> Values, DILocalVariable *Variable,
6317 DIExpression *Expr, DebugLoc DbgLoc) {
6318 if (!Expr->isEntryValue() || !hasSingleElement(Values))
6319 return false;
6320
6321 // These properties are guaranteed by the verifier.
6322 const Argument *Arg = cast<Argument>(Values[0]);
6323 assert(Arg->hasAttribute(Attribute::AttrKind::SwiftAsync));
6324
6325 auto ArgIt = FuncInfo.ValueMap.find(Arg);
6326 if (ArgIt == FuncInfo.ValueMap.end()) {
6327 LLVM_DEBUG(
6328 dbgs() << "Dropping dbg.value: expression is entry_value but "
6329 "couldn't find an associated register for the Argument\n");
6330 return true;
6331 }
6332 Register ArgVReg = ArgIt->getSecond();
6333
6334 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
6335 if (ArgVReg == VirtReg || ArgVReg == PhysReg) {
6337 Variable, Expr, PhysReg, false /*IsIndidrect*/, DbgLoc, SDNodeOrder);
6338 DAG.AddDbgValue(SDV, false /*treat as dbg.declare byval parameter*/);
6339 return true;
6340 }
6341 LLVM_DEBUG(dbgs() << "Dropping dbg.value: expression is entry_value but "
6342 "couldn't find a physical register\n");
6343 return true;
6344}
6345
6346/// Lower the call to the specified intrinsic function.
6347void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I,
6348 unsigned Intrinsic) {
6349 SDLoc sdl = getCurSDLoc();
6350 switch (Intrinsic) {
6351 case Intrinsic::experimental_convergence_anchor:
6352 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ANCHOR, sdl, MVT::Untyped));
6353 break;
6354 case Intrinsic::experimental_convergence_entry:
6355 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_ENTRY, sdl, MVT::Untyped));
6356 break;
6357 case Intrinsic::experimental_convergence_loop: {
6358 auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl);
6359 auto *Token = Bundle->Inputs[0].get();
6360 setValue(&I, DAG.getNode(ISD::CONVERGENCECTRL_LOOP, sdl, MVT::Untyped,
6361 getValue(Token)));
6362 break;
6363 }
6364 }
6365}
6366
6367void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
6368 unsigned IntrinsicID) {
6369 // For now, we're only lowering an 'add' histogram.
6370 // We can add others later, e.g. saturating adds, min/max.
6371 assert(IntrinsicID == Intrinsic::experimental_vector_histogram_add &&
6372 "Tried to lower unsupported histogram type");
6373 SDLoc sdl = getCurSDLoc();
6374 Value *Ptr = I.getOperand(0);
6375 SDValue Inc = getValue(I.getOperand(1));
6376 SDValue Mask = getValue(I.getOperand(2));
6377
6379 DataLayout TargetDL = DAG.getDataLayout();
6380 EVT VT = Inc.getValueType();
6381 Align Alignment = DAG.getEVTAlign(VT);
6382
6383 const MDNode *Ranges = getRangeMetadata(I);
6384
6385 SDValue Root = DAG.getRoot();
6386 SDValue Base;
6387 SDValue Index;
6388 ISD::MemIndexType IndexType;
6389 SDValue Scale;
6390 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
6391 I.getParent(), VT.getScalarStoreSize());
6392
6393 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
6394
6398 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges);
6399
6400 if (!UniformBase) {
6402 Index = getValue(Ptr);
6403 IndexType = ISD::SIGNED_SCALED;
6404 Scale =
6406 }
6407
6408 EVT IdxVT = Index.getValueType();
6409 EVT EltTy = IdxVT.getVectorElementType();
6410 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
6411 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
6412 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index);
6413 }
6414
6415 SDValue ID = DAG.getTargetConstant(IntrinsicID, sdl, MVT::i32);
6416
6417 SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
6418 SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
6419 Ops, MMO, IndexType);
6420
6421 setValue(&I, Histogram);
6422 DAG.setRoot(Histogram);
6423}
6424
6425void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
6426 unsigned Intrinsic) {
6427 assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active &&
6428 "Tried lowering invalid vector extract last");
6429 SDLoc sdl = getCurSDLoc();
6430 const DataLayout &Layout = DAG.getDataLayout();
6431 SDValue Data = getValue(I.getOperand(0));
6432 SDValue Mask = getValue(I.getOperand(1));
6433
6435 EVT ResVT = TLI.getValueType(Layout, I.getType());
6436
6437 EVT ExtVT = TLI.getVectorIdxTy(Layout);
6438 SDValue Idx = DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, sdl, ExtVT, Mask);
6440
6441 Value *Default = I.getOperand(2);
6442 if (!isa<PoisonValue>(Default) && !isa<UndefValue>(Default)) {
6443 SDValue PassThru = getValue(Default);
6444 EVT BoolVT = Mask.getValueType().getScalarType();
6445 SDValue AnyActive = DAG.getNode(ISD::VECREDUCE_OR, sdl, BoolVT, Mask);
6446 Result = DAG.getSelect(sdl, ResVT, AnyActive, Result, PassThru);
6447 }
6448
6449 setValue(&I, Result);
6450}
6451
6452/// Lower the call to the specified intrinsic function.
6453void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
6454 unsigned Intrinsic) {
6456 SDLoc sdl = getCurSDLoc();
6457 DebugLoc dl = getCurDebugLoc();
6458 SDValue Res;
6459
6461 if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
6462 Flags.copyFMF(*FPOp);
6463
6464 switch (Intrinsic) {
6465 default:
6466 // By default, turn this into a target intrinsic node.
6467 visitTargetIntrinsic(I, Intrinsic);
6468 return;
6469 case Intrinsic::vscale: {
6470 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6471 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1)));
6472 return;
6473 }
6474 case Intrinsic::vastart: visitVAStart(I); return;
6475 case Intrinsic::vaend: visitVAEnd(I); return;
6476 case Intrinsic::vacopy: visitVACopy(I); return;
6477 case Intrinsic::returnaddress:
6479 TLI.getValueType(DAG.getDataLayout(), I.getType()),
6480 getValue(I.getArgOperand(0))));
6481 return;
6482 case Intrinsic::addressofreturnaddress:
6483 setValue(&I,
6485 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6486 return;
6487 case Intrinsic::sponentry:
6488 setValue(&I,
6490 TLI.getValueType(DAG.getDataLayout(), I.getType())));
6491 return;
6492 case Intrinsic::frameaddress:
6495 getValue(I.getArgOperand(0))));
6496 return;
6497 case Intrinsic::read_volatile_register:
6498 case Intrinsic::read_register: {
6499 Value *Reg = I.getArgOperand(0);
6500 SDValue Chain = getRoot();
6502 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6503 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6504 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
6505 DAG.getVTList(VT, MVT::Other), Chain, RegName);
6506 setValue(&I, Res);
6507 DAG.setRoot(Res.getValue(1));
6508 return;
6509 }
6510 case Intrinsic::write_register: {
6511 Value *Reg = I.getArgOperand(0);
6512 Value *RegValue = I.getArgOperand(1);
6513 SDValue Chain = getRoot();
6515 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
6516 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
6517 RegName, getValue(RegValue)));
6518 return;
6519 }
6520 case Intrinsic::memcpy: {
6521 const auto &MCI = cast<MemCpyInst>(I);
6522 SDValue Op1 = getValue(I.getArgOperand(0));
6523 SDValue Op2 = getValue(I.getArgOperand(1));
6524 SDValue Op3 = getValue(I.getArgOperand(2));
6525 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
6526 Align DstAlign = MCI.getDestAlign().valueOrOne();
6527 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6528 Align Alignment = std::min(DstAlign, SrcAlign);
6529 bool isVol = MCI.isVolatile();
6530 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6531 // node.
6532 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6533 SDValue MC = DAG.getMemcpy(Root, sdl, Op1, Op2, Op3, Alignment, isVol,
6534 /* AlwaysInline */ false, &I, std::nullopt,
6535 MachinePointerInfo(I.getArgOperand(0)),
6536 MachinePointerInfo(I.getArgOperand(1)),
6537 I.getAAMetadata(), AA);
6538 updateDAGForMaybeTailCall(MC);
6539 return;
6540 }
6541 case Intrinsic::memcpy_inline: {
6542 const auto &MCI = cast<MemCpyInlineInst>(I);
6543 SDValue Dst = getValue(I.getArgOperand(0));
6544 SDValue Src = getValue(I.getArgOperand(1));
6545 SDValue Size = getValue(I.getArgOperand(2));
6546 assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size");
6547 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment.
6548 Align DstAlign = MCI.getDestAlign().valueOrOne();
6549 Align SrcAlign = MCI.getSourceAlign().valueOrOne();
6550 Align Alignment = std::min(DstAlign, SrcAlign);
6551 bool isVol = MCI.isVolatile();
6552 // FIXME: Support passing different dest/src alignments to the memcpy DAG
6553 // node.
6554 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Alignment, isVol,
6555 /* AlwaysInline */ true, &I, std::nullopt,
6556 MachinePointerInfo(I.getArgOperand(0)),
6557 MachinePointerInfo(I.getArgOperand(1)),
6558 I.getAAMetadata(), AA);
6559 updateDAGForMaybeTailCall(MC);
6560 return;
6561 }
6562 case Intrinsic::memset: {
6563 const auto &MSI = cast<MemSetInst>(I);
6564 SDValue Op1 = getValue(I.getArgOperand(0));
6565 SDValue Op2 = getValue(I.getArgOperand(1));
6566 SDValue Op3 = getValue(I.getArgOperand(2));
6567 // @llvm.memset defines 0 and 1 to both mean no alignment.
6568 Align Alignment = MSI.getDestAlign().valueOrOne();
6569 bool isVol = MSI.isVolatile();
6570 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6571 SDValue MS = DAG.getMemset(
6572 Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false,
6573 &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata());
6574 updateDAGForMaybeTailCall(MS);
6575 return;
6576 }
6577 case Intrinsic::memset_inline: {
6578 const auto &MSII = cast<MemSetInlineInst>(I);
6579 SDValue Dst = getValue(I.getArgOperand(0));
6580 SDValue Value = getValue(I.getArgOperand(1));
6581 SDValue Size = getValue(I.getArgOperand(2));
6582 assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size");
6583 // @llvm.memset defines 0 and 1 to both mean no alignment.
6584 Align DstAlign = MSII.getDestAlign().valueOrOne();
6585 bool isVol = MSII.isVolatile();
6586 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6587 SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol,
6588 /* AlwaysInline */ true, &I,
6589 MachinePointerInfo(I.getArgOperand(0)),
6590 I.getAAMetadata());
6591 updateDAGForMaybeTailCall(MC);
6592 return;
6593 }
6594 case Intrinsic::memmove: {
6595 const auto &MMI = cast<MemMoveInst>(I);
6596 SDValue Op1 = getValue(I.getArgOperand(0));
6597 SDValue Op2 = getValue(I.getArgOperand(1));
6598 SDValue Op3 = getValue(I.getArgOperand(2));
6599 // @llvm.memmove defines 0 and 1 to both mean no alignment.
6600 Align DstAlign = MMI.getDestAlign().valueOrOne();
6601 Align SrcAlign = MMI.getSourceAlign().valueOrOne();
6602 Align Alignment = std::min(DstAlign, SrcAlign);
6603 bool isVol = MMI.isVolatile();
6604 // FIXME: Support passing different dest/src alignments to the memmove DAG
6605 // node.
6606 SDValue Root = isVol ? getRoot() : getMemoryRoot();
6607 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, &I,
6608 /* OverrideTailCall */ std::nullopt,
6609 MachinePointerInfo(I.getArgOperand(0)),
6610 MachinePointerInfo(I.getArgOperand(1)),
6611 I.getAAMetadata(), AA);
6612 updateDAGForMaybeTailCall(MM);
6613 return;
6614 }
6615 case Intrinsic::memcpy_element_unordered_atomic: {
6616 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
6617 SDValue Dst = getValue(MI.getRawDest());
6618 SDValue Src = getValue(MI.getRawSource());
6619 SDValue Length = getValue(MI.getLength());
6620
6621 Type *LengthTy = MI.getLength()->getType();
6622 unsigned ElemSz = MI.getElementSizeInBytes();
6623 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6624 SDValue MC =
6625 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6626 isTC, MachinePointerInfo(MI.getRawDest()),
6627 MachinePointerInfo(MI.getRawSource()));
6628 updateDAGForMaybeTailCall(MC);
6629 return;
6630 }
6631 case Intrinsic::memmove_element_unordered_atomic: {
6632 auto &MI = cast<AtomicMemMoveInst>(I);
6633 SDValue Dst = getValue(MI.getRawDest());
6634 SDValue Src = getValue(MI.getRawSource());
6635 SDValue Length = getValue(MI.getLength());
6636
6637 Type *LengthTy = MI.getLength()->getType();
6638 unsigned ElemSz = MI.getElementSizeInBytes();
6639 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6640 SDValue MC =
6641 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz,
6642 isTC, MachinePointerInfo(MI.getRawDest()),
6643 MachinePointerInfo(MI.getRawSource()));
6644 updateDAGForMaybeTailCall(MC);
6645 return;
6646 }
6647 case Intrinsic::memset_element_unordered_atomic: {
6648 auto &MI = cast<AtomicMemSetInst>(I);
6649 SDValue Dst = getValue(MI.getRawDest());
6650 SDValue Val = getValue(MI.getValue());
6651 SDValue Length = getValue(MI.getLength());
6652
6653 Type *LengthTy = MI.getLength()->getType();
6654 unsigned ElemSz = MI.getElementSizeInBytes();
6655 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget());
6656 SDValue MC =
6657 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz,
6658 isTC, MachinePointerInfo(MI.getRawDest()));
6659 updateDAGForMaybeTailCall(MC);
6660 return;
6661 }
6662 case Intrinsic::call_preallocated_setup: {
6663 const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
6664 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6665 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other,
6666 getRoot(), SrcValue);
6667 setValue(&I, Res);
6668 DAG.setRoot(Res);
6669 return;
6670 }
6671 case Intrinsic::call_preallocated_arg: {
6672 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0));
6673 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
6674 SDValue Ops[3];
6675 Ops[0] = getRoot();
6676 Ops[1] = SrcValue;
6677 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
6678 MVT::i32); // arg index
6679 SDValue Res = DAG.getNode(
6681 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops);
6682 setValue(&I, Res);
6683 DAG.setRoot(Res.getValue(1));
6684 return;
6685 }
6686 case Intrinsic::dbg_declare: {
6687 const auto &DI = cast<DbgDeclareInst>(I);
6688 // Debug intrinsics are handled separately in assignment tracking mode.
6689 // Some intrinsics are handled right after Argument lowering.
6690 if (AssignmentTrackingEnabled ||
6692 return;
6693 LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DI << "\n");
6694 DILocalVariable *Variable = DI.getVariable();
6695 DIExpression *Expression = DI.getExpression();
6697 // Assume dbg.declare can not currently use DIArgList, i.e.
6698 // it is non-variadic.
6699 assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
6700 handleDebugDeclare(DI.getVariableLocationOp(0), Variable, Expression,
6701 DI.getDebugLoc());
6702 return;
6703 }
6704 case Intrinsic::dbg_label: {
6705 const DbgLabelInst &DI = cast<DbgLabelInst>(I);
6706 DILabel *Label = DI.getLabel();
6707 assert(Label && "Missing label");
6708
6709 SDDbgLabel *SDV;
6710 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
6711 DAG.AddDbgLabel(SDV);
6712 return;
6713 }
6714 case Intrinsic::dbg_assign: {
6715 // Debug intrinsics are handled separately in assignment tracking mode.
6716 if (AssignmentTrackingEnabled)
6717 return;
6718 // If assignment tracking hasn't been enabled then fall through and treat
6719 // the dbg.assign as a dbg.value.
6720 [[fallthrough]];
6721 }
6722 case Intrinsic::dbg_value: {
6723 // Debug intrinsics are handled separately in assignment tracking mode.
6724 if (AssignmentTrackingEnabled)
6725 return;
6726 const DbgValueInst &DI = cast<DbgValueInst>(I);
6727 assert(DI.getVariable() && "Missing variable");
6728
6729 DILocalVariable *Variable = DI.getVariable();
6732
6733 if (DI.isKillLocation()) {
6734 handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
6735 return;
6736 }
6737
6739 if (Values.empty())
6740 return;
6741
6742 bool IsVariadic = DI.hasArgList();
6743 if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
6744 SDNodeOrder, IsVariadic))
6745 addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
6746 DI.getDebugLoc(), SDNodeOrder);
6747 return;
6748 }
6749
6750 case Intrinsic::eh_typeid_for: {
6751 // Find the type id for the given typeinfo.
6752 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
6753 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
6754 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
6755 setValue(&I, Res);
6756 return;
6757 }
6758
6759 case Intrinsic::eh_return_i32:
6760 case Intrinsic::eh_return_i64:
6763 MVT::Other,
6765 getValue(I.getArgOperand(0)),
6766 getValue(I.getArgOperand(1))));
6767 return;
6768 case Intrinsic::eh_unwind_init:
6770 return;
6771 case Intrinsic::eh_dwarf_cfa:
6774 getValue(I.getArgOperand(0))));
6775 return;
6776 case Intrinsic::eh_sjlj_callsite: {
6777 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0));
6778 assert(FuncInfo.getCurrentCallSite() == 0 && "Overlapping call sites!");
6779
6781 return;
6782 }
6783 case Intrinsic::eh_sjlj_functioncontext: {
6784 // Get and store the index of the function context.
6786 AllocaInst *FnCtx =
6787 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
6788 int FI = FuncInfo.StaticAllocaMap[FnCtx];
6790 return;
6791 }
6792 case Intrinsic::eh_sjlj_setjmp: {
6793 SDValue Ops[2];
6794 Ops[0] = getRoot();
6795 Ops[1] = getValue(I.getArgOperand(0));
6797 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6798 setValue(&I, Op.getValue(0));
6799 DAG.setRoot(Op.getValue(1));
6800 return;
6801 }
6802 case Intrinsic::eh_sjlj_longjmp:
6803 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
6804 getRoot(), getValue(I.getArgOperand(0))));
6805 return;
6806 case Intrinsic::eh_sjlj_setup_dispatch:
6808 getRoot()));
6809 return;
6810 case Intrinsic::masked_gather:
6811 visitMaskedGather(I);
6812 return;
6813 case Intrinsic::masked_load:
6814 visitMaskedLoad(I);
6815 return;
6816 case Intrinsic::masked_scatter:
6817 visitMaskedScatter(I);
6818 return;
6819 case Intrinsic::masked_store:
6820 visitMaskedStore(I);
6821 return;
6822 case Intrinsic::masked_expandload:
6823 visitMaskedLoad(I, true /* IsExpanding */);
6824 return;
6825 case Intrinsic::masked_compressstore:
6826 visitMaskedStore(I, true /* IsCompressing */);
6827 return;
6828 case Intrinsic::powi:
6829 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
6830 getValue(I.getArgOperand(1)), DAG));
6831 return;
6832 case Intrinsic::log:
6833 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6834 return;
6835 case Intrinsic::log2:
6836 setValue(&I,
6837 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6838 return;
6839 case Intrinsic::log10:
6840 setValue(&I,
6841 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6842 return;
6843 case Intrinsic::exp:
6844 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6845 return;
6846 case Intrinsic::exp2:
6847 setValue(&I,
6848 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags));
6849 return;
6850 case Intrinsic::pow:
6851 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
6852 getValue(I.getArgOperand(1)), DAG, TLI, Flags));
6853 return;
6854 case Intrinsic::sqrt:
6855 case Intrinsic::fabs:
6856 case Intrinsic::sin:
6857 case Intrinsic::cos:
6858 case Intrinsic::tan:
6859 case Intrinsic::asin:
6860 case Intrinsic::acos:
6861 case Intrinsic::atan:
6862 case Intrinsic::sinh:
6863 case Intrinsic::cosh:
6864 case Intrinsic::tanh:
6865 case Intrinsic::exp10:
6866 case Intrinsic::floor:
6867 case Intrinsic::ceil:
6868 case Intrinsic::trunc:
6869 case Intrinsic::rint:
6870 case Intrinsic::nearbyint:
6871 case Intrinsic::round:
6872 case Intrinsic::roundeven:
6873 case Intrinsic::canonicalize: {
6874 unsigned Opcode;
6875 // clang-format off
6876 switch (Intrinsic) {
6877 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6878 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
6879 case Intrinsic::fabs: Opcode = ISD::FABS; break;
6880 case Intrinsic::sin: Opcode = ISD::FSIN; break;
6881 case Intrinsic::cos: Opcode = ISD::FCOS; break;
6882 case Intrinsic::tan: Opcode = ISD::FTAN; break;
6883 case Intrinsic::asin: Opcode = ISD::FASIN; break;
6884 case Intrinsic::acos: Opcode = ISD::FACOS; break;
6885 case Intrinsic::atan: Opcode = ISD::FATAN; break;
6886 case Intrinsic::sinh: Opcode = ISD::FSINH; break;
6887 case Intrinsic::cosh: Opcode = ISD::FCOSH; break;
6888 case Intrinsic::tanh: Opcode = ISD::FTANH; break;
6889 case Intrinsic::exp10: Opcode = ISD::FEXP10; break;
6890 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
6891 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
6892 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
6893 case Intrinsic::rint: Opcode = ISD::FRINT; break;
6894 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
6895 case Intrinsic::round: Opcode = ISD::FROUND; break;
6896 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break;
6897 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
6898 }
6899 // clang-format on
6900
6901 setValue(&I, DAG.getNode(Opcode, sdl,
6902 getValue(I.getArgOperand(0)).getValueType(),
6903 getValue(I.getArgOperand(0)), Flags));
6904 return;
6905 }
6906 case Intrinsic::atan2:
6908 getValue(I.getArgOperand(0)).getValueType(),
6909 getValue(I.getArgOperand(0)),
6910 getValue(I.getArgOperand(1)), Flags));
6911 return;
6912 case Intrinsic::lround:
6913 case Intrinsic::llround:
6914 case Intrinsic::lrint:
6915 case Intrinsic::llrint: {
6916 unsigned Opcode;
6917 // clang-format off
6918 switch (Intrinsic) {
6919 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
6920 case Intrinsic::lround: Opcode = ISD::LROUND; break;
6921 case Intrinsic::llround: Opcode = ISD::LLROUND; break;
6922 case Intrinsic::lrint: Opcode = ISD::LRINT; break;
6923 case Intrinsic::llrint: Opcode = ISD::LLRINT; break;
6924 }
6925 // clang-format on
6926
6927 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
6928 setValue(&I, DAG.getNode(Opcode, sdl, RetVT,
6929 getValue(I.getArgOperand(0))));
6930 return;
6931 }
6932 case Intrinsic::minnum:
6934 getValue(I.getArgOperand(0)).getValueType(),
6935 getValue(I.getArgOperand(0)),
6936 getValue(I.getArgOperand(1)), Flags));
6937 return;
6938 case Intrinsic::maxnum:
6940 getValue(I.getArgOperand(0)).getValueType(),
6941 getValue(I.getArgOperand(0)),
6942 getValue(I.getArgOperand(1)), Flags));
6943 return;
6944 case Intrinsic::minimum:
6946 getValue(I.getArgOperand(0)).getValueType(),
6947 getValue(I.getArgOperand(0)),
6948 getValue(I.getArgOperand(1)), Flags));
6949 return;
6950 case Intrinsic::maximum:
6952 getValue(I.getArgOperand(0)).getValueType(),
6953 getValue(I.getArgOperand(0)),
6954 getValue(I.getArgOperand(1)), Flags));
6955 return;
6956 case Intrinsic::minimumnum:
6958 getValue(I.getArgOperand(0)).getValueType(),
6959 getValue(I.getArgOperand(0)),
6960 getValue(I.getArgOperand(1)), Flags));
6961 return;
6962 case Intrinsic::maximumnum:
6964 getValue(I.getArgOperand(0)).getValueType(),
6965 getValue(I.getArgOperand(0)),
6966 getValue(I.getArgOperand(1)), Flags));
6967 return;
6968 case Intrinsic::copysign:
6970 getValue(I.getArgOperand(0)).getValueType(),
6971 getValue(I.getArgOperand(0)),
6972 getValue(I.getArgOperand(1)), Flags));
6973 return;
6974 case Intrinsic::ldexp:
6976 getValue(I.getArgOperand(0)).getValueType(),
6977 getValue(I.getArgOperand(0)),
6978 getValue(I.getArgOperand(1)), Flags));
6979 return;
6980 case Intrinsic::sincos:
6981 case Intrinsic::frexp: {
6982 unsigned Opcode;
6983 switch (Intrinsic) {
6984 default:
6985 llvm_unreachable("unexpected intrinsic");
6986 case Intrinsic::sincos:
6987 Opcode = ISD::FSINCOS;
6988 break;
6989 case Intrinsic::frexp:
6990 Opcode = ISD::FFREXP;
6991 break;
6992 }
6993 SmallVector<EVT, 2> ValueVTs;
6994 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
6995 SDVTList VTs = DAG.getVTList(ValueVTs);
6996 setValue(
6997 &I, DAG.getNode(Opcode, sdl, VTs, getValue(I.getArgOperand(0)), Flags));
6998 return;
6999 }
7000 case Intrinsic::arithmetic_fence: {
7002 getValue(I.getArgOperand(0)).getValueType(),
7003 getValue(I.getArgOperand(0)), Flags));
7004 return;
7005 }
7006 case Intrinsic::fma:
7008 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(),
7009 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)),
7010 getValue(I.getArgOperand(2)), Flags));
7011 return;
7012#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \
7013 case Intrinsic::INTRINSIC:
7014#include "llvm/IR/ConstrainedOps.def"
7015 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
7016 return;
7017#define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID:
7018#include "llvm/IR/VPIntrinsics.def"
7019 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I));
7020 return;
7021 case Intrinsic::fptrunc_round: {
7022 // Get the last argument, the metadata and convert it to an integer in the
7023 // call
7024 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata();
7025 std::optional<RoundingMode> RoundMode =
7026 convertStrToRoundingMode(cast<MDString>(MD)->getString());
7027
7028 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7029
7030 // Propagate fast-math-flags from IR to node(s).
7032 Flags.copyFMF(*cast<FPMathOperator>(&I));
7033 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
7034
7036 Result = DAG.getNode(
7037 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)),
7038 DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32));
7039 setValue(&I, Result);
7040
7041 return;
7042 }
7043 case Intrinsic::fmuladd: {
7044 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7047 setValue(&I, DAG.getNode(ISD::FMA, sdl,
7048 getValue(I.getArgOperand(0)).getValueType(),
7049 getValue(I.getArgOperand(0)),
7050 getValue(I.getArgOperand(1)),
7051 getValue(I.getArgOperand(2)), Flags));
7052 } else {
7053 // TODO: Intrinsic calls should have fast-math-flags.
7055 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(),
7056 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags);
7058 getValue(I.getArgOperand(0)).getValueType(),
7059 Mul, getValue(I.getArgOperand(2)), Flags);
7060 setValue(&I, Add);
7061 }
7062 return;
7063 }
7064 case Intrinsic::convert_to_fp16:
7065 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
7066 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
7067 getValue(I.getArgOperand(0)),
7068 DAG.getTargetConstant(0, sdl,
7069 MVT::i32))));
7070 return;
7071 case Intrinsic::convert_from_fp16:
7073 TLI.getValueType(DAG.getDataLayout(), I.getType()),
7074 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
7075 getValue(I.getArgOperand(0)))));
7076 return;
7077 case Intrinsic::fptosi_sat: {
7078 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7080 getValue(I.getArgOperand(0)),
7082 return;
7083 }
7084 case Intrinsic::fptoui_sat: {
7085 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7087 getValue(I.getArgOperand(0)),
7089 return;
7090 }
7091 case Intrinsic::set_rounding:
7092 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other,
7093 {getRoot(), getValue(I.getArgOperand(0))});
7094 setValue(&I, Res);
7095 DAG.setRoot(Res.getValue(0));
7096 return;
7097 case Intrinsic::is_fpclass: {
7098 const DataLayout DLayout = DAG.getDataLayout();
7099 EVT DestVT = TLI.getValueType(DLayout, I.getType());
7100 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType());
7101 FPClassTest Test = static_cast<FPClassTest>(
7102 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
7104 const Function &F = MF.getFunction();
7105 SDValue Op = getValue(I.getArgOperand(0));
7107 Flags.setNoFPExcept(
7108 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP));
7109 // If ISD::IS_FPCLASS should be expanded, do it right now, because the
7110 // expansion can use illegal types. Making expansion early allows
7111 // legalizing these types prior to selection.
7112 if (!TLI.isOperationLegal(ISD::IS_FPCLASS, ArgVT) &&
7113 !TLI.isOperationCustom(ISD::IS_FPCLASS, ArgVT)) {
7114 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG);
7115 setValue(&I, Result);
7116 return;
7117 }
7118
7119 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32);
7120 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags);
7121 setValue(&I, V);
7122 return;
7123 }
7124 case Intrinsic::get_fpenv: {
7125 const DataLayout DLayout = DAG.getDataLayout();
7126 EVT EnvVT = TLI.getValueType(DLayout, I.getType());
7127 Align TempAlign = DAG.getEVTAlign(EnvVT);
7128 SDValue Chain = getRoot();
7129 // Use GET_FPENV if it is legal or custom. Otherwise use memory-based node
7130 // and temporary storage in stack.
7131 if (TLI.isOperationLegalOrCustom(ISD::GET_FPENV, EnvVT)) {
7132 Res = DAG.getNode(
7133 ISD::GET_FPENV, sdl,
7134 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7135 MVT::Other),
7136 Chain);
7137 } else {
7138 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7139 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7140 auto MPI =
7144 TempAlign);
7145 Chain = DAG.getGetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7146 Res = DAG.getLoad(EnvVT, sdl, Chain, Temp, MPI);
7147 }
7148 setValue(&I, Res);
7149 DAG.setRoot(Res.getValue(1));
7150 return;
7151 }
7152 case Intrinsic::set_fpenv: {
7153 const DataLayout DLayout = DAG.getDataLayout();
7154 SDValue Env = getValue(I.getArgOperand(0));
7155 EVT EnvVT = Env.getValueType();
7156 Align TempAlign = DAG.getEVTAlign(EnvVT);
7157 SDValue Chain = getRoot();
7158 // If SET_FPENV is custom or legal, use it. Otherwise use loading
7159 // environment from memory.
7160 if (TLI.isOperationLegalOrCustom(ISD::SET_FPENV, EnvVT)) {
7161 Chain = DAG.getNode(ISD::SET_FPENV, sdl, MVT::Other, Chain, Env);
7162 } else {
7163 // Allocate space in stack, copy environment bits into it and use this
7164 // memory in SET_FPENV_MEM.
7165 SDValue Temp = DAG.CreateStackTemporary(EnvVT, TempAlign.value());
7166 int SPFI = cast<FrameIndexSDNode>(Temp.getNode())->getIndex();
7167 auto MPI =
7169 Chain = DAG.getStore(Chain, sdl, Env, Temp, MPI, TempAlign,
7173 TempAlign);
7174 Chain = DAG.getSetFPEnv(Chain, sdl, Temp, EnvVT, MMO);
7175 }
7176 DAG.setRoot(Chain);
7177 return;
7178 }
7179 case Intrinsic::reset_fpenv:
7180 DAG.setRoot(DAG.getNode(ISD::RESET_FPENV, sdl, MVT::Other, getRoot()));
7181 return;
7182 case Intrinsic::get_fpmode:
7183 Res = DAG.getNode(
7184 ISD::GET_FPMODE, sdl,
7185 DAG.getVTList(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7186 MVT::Other),
7187 DAG.getRoot());
7188 setValue(&I, Res);
7189 DAG.setRoot(Res.getValue(1));
7190 return;
7191 case Intrinsic::set_fpmode:
7192 Res = DAG.getNode(ISD::SET_FPMODE, sdl, MVT::Other, {DAG.getRoot()},
7193 getValue(I.getArgOperand(0)));
7194 DAG.setRoot(Res);
7195 return;
7196 case Intrinsic::reset_fpmode: {
7197 Res = DAG.getNode(ISD::RESET_FPMODE, sdl, MVT::Other, getRoot());
7198 DAG.setRoot(Res);
7199 return;
7200 }
7201 case Intrinsic::pcmarker: {
7202 SDValue Tmp = getValue(I.getArgOperand(0));
7203 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
7204 return;
7205 }
7206 case Intrinsic::readcyclecounter: {
7207 SDValue Op = getRoot();
7209 DAG.getVTList(MVT::i64, MVT::Other), Op);
7210 setValue(&I, Res);
7211 DAG.setRoot(Res.getValue(1));
7212 return;
7213 }
7214 case Intrinsic::readsteadycounter: {
7215 SDValue Op = getRoot();
7217 DAG.getVTList(MVT::i64, MVT::Other), Op);
7218 setValue(&I, Res);
7219 DAG.setRoot(Res.getValue(1));
7220 return;
7221 }
7222 case Intrinsic::bitreverse:
7224 getValue(I.getArgOperand(0)).getValueType(),
7225 getValue(I.getArgOperand(0))));
7226 return;
7227 case Intrinsic::bswap:
7229 getValue(I.getArgOperand(0)).getValueType(),
7230 getValue(I.getArgOperand(0))));
7231 return;
7232 case Intrinsic::cttz: {
7233 SDValue Arg = getValue(I.getArgOperand(0));
7234 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7235 EVT Ty = Arg.getValueType();
7237 sdl, Ty, Arg));
7238 return;
7239 }
7240 case Intrinsic::ctlz: {
7241 SDValue Arg = getValue(I.getArgOperand(0));
7242 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
7243 EVT Ty = Arg.getValueType();
7245 sdl, Ty, Arg));
7246 return;
7247 }
7248 case Intrinsic::ctpop: {
7249 SDValue Arg = getValue(I.getArgOperand(0));
7250 EVT Ty = Arg.getValueType();
7251 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
7252 return;
7253 }
7254 case Intrinsic::fshl:
7255 case Intrinsic::fshr: {
7256 bool IsFSHL = Intrinsic == Intrinsic::fshl;
7257 SDValue X = getValue(I.getArgOperand(0));
7258 SDValue Y = getValue(I.getArgOperand(1));
7259 SDValue Z = getValue(I.getArgOperand(2));
7260 EVT VT = X.getValueType();
7261
7262 if (X == Y) {
7263 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
7264 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
7265 } else {
7266 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR;
7267 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z));
7268 }
7269 return;
7270 }
7271 case Intrinsic::sadd_sat: {
7272 SDValue Op1 = getValue(I.getArgOperand(0));
7273 SDValue Op2 = getValue(I.getArgOperand(1));
7274 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7275 return;
7276 }
7277 case Intrinsic::uadd_sat: {
7278 SDValue Op1 = getValue(I.getArgOperand(0));
7279 SDValue Op2 = getValue(I.getArgOperand(1));
7280 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
7281 return;
7282 }
7283 case Intrinsic::ssub_sat: {
7284 SDValue Op1 = getValue(I.getArgOperand(0));
7285 SDValue Op2 = getValue(I.getArgOperand(1));
7286 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7287 return;
7288 }
7289 case Intrinsic::usub_sat: {
7290 SDValue Op1 = getValue(I.getArgOperand(0));
7291 SDValue Op2 = getValue(I.getArgOperand(1));
7292 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
7293 return;
7294 }
7295 case Intrinsic::sshl_sat: {
7296 SDValue Op1 = getValue(I.getArgOperand(0));
7297 SDValue Op2 = getValue(I.getArgOperand(1));
7298 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7299 return;
7300 }
7301 case Intrinsic::ushl_sat: {
7302 SDValue Op1 = getValue(I.getArgOperand(0));
7303 SDValue Op2 = getValue(I.getArgOperand(1));
7304 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2));
7305 return;
7306 }
7307 case Intrinsic::smul_fix:
7308 case Intrinsic::umul_fix:
7309 case Intrinsic::smul_fix_sat:
7310 case Intrinsic::umul_fix_sat: {
7311 SDValue Op1 = getValue(I.getArgOperand(0));
7312 SDValue Op2 = getValue(I.getArgOperand(1));
7313 SDValue Op3 = getValue(I.getArgOperand(2));
7315 Op1.getValueType(), Op1, Op2, Op3));
7316 return;
7317 }
7318 case Intrinsic::sdiv_fix:
7319 case Intrinsic::udiv_fix:
7320 case Intrinsic::sdiv_fix_sat:
7321 case Intrinsic::udiv_fix_sat: {
7322 SDValue Op1 = getValue(I.getArgOperand(0));
7323 SDValue Op2 = getValue(I.getArgOperand(1));
7324 SDValue Op3 = getValue(I.getArgOperand(2));
7326 Op1, Op2, Op3, DAG, TLI));
7327 return;
7328 }
7329 case Intrinsic::smax: {
7330 SDValue Op1 = getValue(I.getArgOperand(0));
7331 SDValue Op2 = getValue(I.getArgOperand(1));
7332 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2));
7333 return;
7334 }
7335 case Intrinsic::smin: {
7336 SDValue Op1 = getValue(I.getArgOperand(0));
7337 SDValue Op2 = getValue(I.getArgOperand(1));
7338 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2));
7339 return;
7340 }
7341 case Intrinsic::umax: {
7342 SDValue Op1 = getValue(I.getArgOperand(0));
7343 SDValue Op2 = getValue(I.getArgOperand(1));
7344 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2));
7345 return;
7346 }
7347 case Intrinsic::umin: {
7348 SDValue Op1 = getValue(I.getArgOperand(0));
7349 SDValue Op2 = getValue(I.getArgOperand(1));
7350 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2));
7351 return;
7352 }
7353 case Intrinsic::abs: {
7354 // TODO: Preserve "int min is poison" arg in SDAG?
7355 SDValue Op1 = getValue(I.getArgOperand(0));
7356 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
7357 return;
7358 }
7359 case Intrinsic::scmp: {
7360 SDValue Op1 = getValue(I.getArgOperand(0));
7361 SDValue Op2 = getValue(I.getArgOperand(1));
7362 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7363 setValue(&I, DAG.getNode(ISD::SCMP, sdl, DestVT, Op1, Op2));
7364 break;
7365 }
7366 case Intrinsic::ucmp: {
7367 SDValue Op1 = getValue(I.getArgOperand(0));
7368 SDValue Op2 = getValue(I.getArgOperand(1));
7369 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7370 setValue(&I, DAG.getNode(ISD::UCMP, sdl, DestVT, Op1, Op2));
7371 break;
7372 }
7373 case Intrinsic::stacksave: {
7374 SDValue Op = getRoot();
7375 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
7376 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op);
7377 setValue(&I, Res);
7378 DAG.setRoot(Res.getValue(1));
7379 return;
7380 }
7381 case Intrinsic::stackrestore:
7382 Res = getValue(I.getArgOperand(0));
7383 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
7384 return;
7385 case Intrinsic::get_dynamic_area_offset: {
7386 SDValue Op = getRoot();
7387 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7388 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7389 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
7390 // target.
7391 if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits())
7392 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
7393 " intrinsic!");
7395 Op);
7396 DAG.setRoot(Op);
7397 setValue(&I, Res);
7398 return;
7399 }
7400 case Intrinsic::stackguard: {
7402 const Module &M = *MF.getFunction().getParent();
7403 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
7404 SDValue Chain = getRoot();
7405 if (TLI.useLoadStackGuardNode(M)) {
7406 Res = getLoadStackGuard(DAG, sdl, Chain);
7407 Res = DAG.getPtrExtOrTrunc(Res, sdl, PtrTy);
7408 } else {
7409 const Value *Global = TLI.getSDagStackGuard(M);
7411 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
7414 }
7415 if (TLI.useStackGuardXorFP())
7416 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
7417 DAG.setRoot(Chain);
7418 setValue(&I, Res);
7419 return;
7420 }
7421 case Intrinsic::stackprotector: {
7422 // Emit code into the DAG to store the stack guard onto the stack.
7424 MachineFrameInfo &MFI = MF.getFrameInfo();
7425 const Module &M = *MF.getFunction().getParent();
7426 SDValue Src, Chain = getRoot();
7427
7428 if (TLI.useLoadStackGuardNode(M))
7429 Src = getLoadStackGuard(DAG, sdl, Chain);
7430 else
7431 Src = getValue(I.getArgOperand(0)); // The guard's value.
7432
7433 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
7434
7435 int FI = FuncInfo.StaticAllocaMap[Slot];
7436 MFI.setStackProtectorIndex(FI);
7437 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout());
7438
7439 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
7440
7441 // Store the stack protector onto the stack.
7442 Res = DAG.getStore(
7443 Chain, sdl, Src, FIN,
7446 setValue(&I, Res);
7447 DAG.setRoot(Res);
7448 return;
7449 }
7450 case Intrinsic::objectsize:
7451 llvm_unreachable("llvm.objectsize.* should have been lowered already");
7452
7453 case Intrinsic::is_constant:
7454 llvm_unreachable("llvm.is.constant.* should have been lowered already");
7455
7456 case Intrinsic::annotation:
7457 case Intrinsic::ptr_annotation:
7458 case Intrinsic::launder_invariant_group:
7459 case Intrinsic::strip_invariant_group:
7460 // Drop the intrinsic, but forward the value
7461 setValue(&I, getValue(I.getOperand(0)));
7462 return;
7463
7464 case Intrinsic::assume:
7465 case Intrinsic::experimental_noalias_scope_decl:
7466 case Intrinsic::var_annotation:
7467 case Intrinsic::sideeffect:
7468 // Discard annotate attributes, noalias scope declarations, assumptions, and
7469 // artificial side-effects.
7470 return;
7471
7472 case Intrinsic::codeview_annotation: {
7473 // Emit a label associated with this metadata.
7475 MCSymbol *Label = MF.getContext().createTempSymbol("annotation", true);
7476 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
7477 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
7478 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
7479 DAG.setRoot(Res);
7480 return;
7481 }
7482
7483 case Intrinsic::init_trampoline: {
7484 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
7485
7486 SDValue Ops[6];
7487 Ops[0] = getRoot();
7488 Ops[1] = getValue(I.getArgOperand(0));
7489 Ops[2] = getValue(I.getArgOperand(1));
7490 Ops[3] = getValue(I.getArgOperand(2));
7491 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
7492 Ops[5] = DAG.getSrcValue(F);
7493
7494 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
7495
7496 DAG.setRoot(Res);
7497 return;
7498 }
7499 case Intrinsic::adjust_trampoline:
7502 getValue(I.getArgOperand(0))));
7503 return;
7504 case Intrinsic::gcroot: {
7506 "only valid in functions with gc specified, enforced by Verifier");
7507 assert(GFI && "implied by previous");
7508 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
7509 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
7510
7511 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
7512 GFI->addStackRoot(FI->getIndex(), TypeMap);
7513 return;
7514 }
7515 case Intrinsic::gcread:
7516 case Intrinsic::gcwrite:
7517 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!");
7518 case Intrinsic::get_rounding:
7519 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot());
7520 setValue(&I, Res);
7521 DAG.setRoot(Res.getValue(1));
7522 return;
7523
7524 case Intrinsic::expect:
7525 case Intrinsic::expect_with_probability:
7526 // Just replace __builtin_expect(exp, c) and
7527 // __builtin_expect_with_probability(exp, c, p) with EXP.
7528 setValue(&I, getValue(I.getArgOperand(0)));
7529 return;
7530
7531 case Intrinsic::ubsantrap:
7532 case Intrinsic::debugtrap:
7533 case Intrinsic::trap: {
7534 StringRef TrapFuncName =
7535 I.getAttributes().getFnAttr("trap-func-name").getValueAsString();
7536 if (TrapFuncName.empty()) {
7537 switch (Intrinsic) {
7538 case Intrinsic::trap:
7539 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot()));
7540 break;
7541 case Intrinsic::debugtrap:
7542 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot()));
7543 break;
7544 case Intrinsic::ubsantrap:
7546 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(),
7548 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl,
7549 MVT::i32)));
7550 break;
7551 default: llvm_unreachable("unknown trap intrinsic");
7552 }
7554 I.hasFnAttr(Attribute::NoMerge));
7555 return;
7556 }
7558 if (Intrinsic == Intrinsic::ubsantrap) {
7560 Args[0].Val = I.getArgOperand(0);
7561 Args[0].Node = getValue(Args[0].Val);
7562 Args[0].Ty = Args[0].Val->getType();
7563 }
7564
7566 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
7567 CallingConv::C, I.getType(),
7568 DAG.getExternalSymbol(TrapFuncName.data(),
7570 std::move(Args));
7571 CLI.NoMerge = I.hasFnAttr(Attribute::NoMerge);
7572 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
7573 DAG.setRoot(Result.second);
7574 return;
7575 }
7576
7577 case Intrinsic::allow_runtime_check:
7578 case Intrinsic::allow_ubsan_check:
7579 setValue(&I, getValue(ConstantInt::getTrue(I.getType())));
7580 return;
7581
7582 case Intrinsic::uadd_with_overflow:
7583 case Intrinsic::sadd_with_overflow:
7584 case Intrinsic::usub_with_overflow:
7585 case Intrinsic::ssub_with_overflow:
7586 case Intrinsic::umul_with_overflow:
7587 case Intrinsic::smul_with_overflow: {
7589 switch (Intrinsic) {
7590 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
7591 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
7592 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
7593 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
7594 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
7595 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
7596 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
7597 }
7598 SDValue Op1 = getValue(I.getArgOperand(0));
7599 SDValue Op2 = getValue(I.getArgOperand(1));
7600
7601 EVT ResultVT = Op1.getValueType();
7602 EVT OverflowVT = MVT::i1;
7603 if (ResultVT.isVector())
7604 OverflowVT = EVT::getVectorVT(
7605 *Context, OverflowVT, ResultVT.getVectorElementCount());
7606
7607 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
7608 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
7609 return;
7610 }
7611 case Intrinsic::prefetch: {
7612 SDValue Ops[5];
7613 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7615 Ops[0] = DAG.getRoot();
7616 Ops[1] = getValue(I.getArgOperand(0));
7617 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl,
7618 MVT::i32);
7619 Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl,
7620 MVT::i32);
7621 Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl,
7622 MVT::i32);
7624 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops,
7625 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)),
7626 /* align */ std::nullopt, Flags);
7627
7628 // Chain the prefetch in parallel with any pending loads, to stay out of
7629 // the way of later optimizations.
7630 PendingLoads.push_back(Result);
7631 Result = getRoot();
7632 DAG.setRoot(Result);
7633 return;
7634 }
7635 case Intrinsic::lifetime_start:
7636 case Intrinsic::lifetime_end: {
7637 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
7638 // Stack coloring is not enabled in O0, discard region information.
7640 return;
7641
7642 const int64_t ObjectSize =
7643 cast<ConstantInt>(I.getArgOperand(0))->getSExtValue();
7644 Value *const ObjectPtr = I.getArgOperand(1);
7646 getUnderlyingObjects(ObjectPtr, Allocas);
7647
7648 for (const Value *Alloca : Allocas) {
7649 const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca);
7650
7651 // Could not find an Alloca.
7652 if (!LifetimeObject)
7653 continue;
7654
7655 // First check that the Alloca is static, otherwise it won't have a
7656 // valid frame index.
7657 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
7658 if (SI == FuncInfo.StaticAllocaMap.end())
7659 return;
7660
7661 const int FrameIndex = SI->second;
7662 int64_t Offset;
7664 ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject)
7665 Offset = -1; // Cannot determine offset from alloca to lifetime object.
7666 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize,
7667 Offset);
7668 DAG.setRoot(Res);
7669 }
7670 return;
7671 }
7672 case Intrinsic::pseudoprobe: {
7673 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue();
7674 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
7675 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
7676 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr);
7677 DAG.setRoot(Res);
7678 return;
7679 }
7680 case Intrinsic::invariant_start:
7681 // Discard region information.
7682 setValue(&I,
7683 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType())));
7684 return;
7685 case Intrinsic::invariant_end:
7686 // Discard region information.
7687 return;
7688 case Intrinsic::clear_cache: {
7689 SDValue InputChain = DAG.getRoot();
7690 SDValue StartVal = getValue(I.getArgOperand(0));
7691 SDValue EndVal = getValue(I.getArgOperand(1));
7692 Res = DAG.getNode(ISD::CLEAR_CACHE, sdl, DAG.getVTList(MVT::Other),
7693 {InputChain, StartVal, EndVal});
7694 setValue(&I, Res);
7695 DAG.setRoot(Res);
7696 return;
7697 }
7698 case Intrinsic::donothing:
7699 case Intrinsic::seh_try_begin:
7700 case Intrinsic::seh_scope_begin:
7701 case Intrinsic::seh_try_end:
7702 case Intrinsic::seh_scope_end:
7703 // ignore
7704 return;
7705 case Intrinsic::experimental_stackmap:
7706 visitStackmap(I);
7707 return;
7708 case Intrinsic::experimental_patchpoint_void:
7709 case Intrinsic::experimental_patchpoint:
7710 visitPatchpoint(I);
7711 return;
7712 case Intrinsic::experimental_gc_statepoint:
7713 LowerStatepoint(cast<GCStatepointInst>(I));
7714 return;
7715 case Intrinsic::experimental_gc_result:
7716 visitGCResult(cast<GCResultInst>(I));
7717 return;
7718 case Intrinsic::experimental_gc_relocate:
7719 visitGCRelocate(cast<GCRelocateInst>(I));
7720 return;
7721 case Intrinsic::instrprof_cover:
7722 llvm_unreachable("instrprof failed to lower a cover");
7723 case Intrinsic::instrprof_increment:
7724 llvm_unreachable("instrprof failed to lower an increment");
7725 case Intrinsic::instrprof_timestamp:
7726 llvm_unreachable("instrprof failed to lower a timestamp");
7727 case Intrinsic::instrprof_value_profile:
7728 llvm_unreachable("instrprof failed to lower a value profiling call");
7729 case Intrinsic::instrprof_mcdc_parameters:
7730 llvm_unreachable("instrprof failed to lower mcdc parameters");
7731 case Intrinsic::instrprof_mcdc_tvbitmap_update:
7732 llvm_unreachable("instrprof failed to lower an mcdc tvbitmap update");
7733 case Intrinsic::localescape: {
7736
7737 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
7738 // is the same on all targets.
7739 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) {
7740 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
7741 if (isa<ConstantPointerNull>(Arg))
7742 continue; // Skip null pointers. They represent a hole in index space.
7743 AllocaInst *Slot = cast<AllocaInst>(Arg);
7744 assert(FuncInfo.StaticAllocaMap.count(Slot) &&
7745 "can only escape static allocas");
7746 int FI = FuncInfo.StaticAllocaMap[Slot];
7747 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7750 TII->get(TargetOpcode::LOCAL_ESCAPE))
7751 .addSym(FrameAllocSym)
7752 .addFrameIndex(FI);
7753 }
7754
7755 return;
7756 }
7757
7758 case Intrinsic::localrecover: {
7759 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
7761
7762 // Get the symbol that defines the frame offset.
7763 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
7764 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
7765 unsigned IdxVal =
7766 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
7767 MCSymbol *FrameAllocSym = MF.getContext().getOrCreateFrameAllocSymbol(
7769
7770 Value *FP = I.getArgOperand(1);
7771 SDValue FPVal = getValue(FP);
7772 EVT PtrVT = FPVal.getValueType();
7773
7774 // Create a MCSymbol for the label to avoid any target lowering
7775 // that would make this PC relative.
7776 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
7777 SDValue OffsetVal =
7778 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
7779
7780 // Add the offset to the FP.
7781 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl);
7782 setValue(&I, Add);
7783
7784 return;
7785 }
7786
7787 case Intrinsic::fake_use: {
7788 Value *V = I.getArgOperand(0);
7789 SDValue Ops[2];
7790 // For Values not declared or previously used in this basic block, the
7791 // NodeMap will not have an entry, and `getValue` will assert if V has no
7792 // valid register value.
7793 auto FakeUseValue = [&]() -> SDValue {
7794 SDValue &N = NodeMap[V];
7795 if (N.getNode())
7796 return N;
7797
7798 // If there's a virtual register allocated and initialized for this
7799 // value, use it.
7800 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
7801 return copyFromReg;
7802 // FIXME: Do we want to preserve constants? It seems pointless.
7803 if (isa<Constant>(V))
7804 return getValue(V);
7805 return SDValue();
7806 }();
7807 if (!FakeUseValue || FakeUseValue.isUndef())
7808 return;
7809 Ops[0] = getRoot();
7810 Ops[1] = FakeUseValue;
7811 // Also, do not translate a fake use with an undef operand, or any other
7812 // empty SDValues.
7813 if (!Ops[1] || Ops[1].isUndef())
7814 return;
7815 DAG.setRoot(DAG.getNode(ISD::FAKE_USE, sdl, MVT::Other, Ops));
7816 return;
7817 }
7818
7819 case Intrinsic::eh_exceptionpointer:
7820 case Intrinsic::eh_exceptioncode: {
7821 // Get the exception pointer vreg, copy from it, and resize it to fit.
7822 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
7823 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
7824 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
7826 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT);
7827 if (Intrinsic == Intrinsic::eh_exceptioncode)
7828 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32);
7829 setValue(&I, N);
7830 return;
7831 }
7832 case Intrinsic::xray_customevent: {
7833 // Here we want to make sure that the intrinsic behaves as if it has a
7834 // specific calling convention.
7835 const auto &Triple = DAG.getTarget().getTargetTriple();
7836 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7837 return;
7838
7840
7841 // We want to say that we always want the arguments in registers.
7842 SDValue LogEntryVal = getValue(I.getArgOperand(0));
7843 SDValue StrSizeVal = getValue(I.getArgOperand(1));
7844 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7845 SDValue Chain = getRoot();
7846 Ops.push_back(LogEntryVal);
7847 Ops.push_back(StrSizeVal);
7848 Ops.push_back(Chain);
7849
7850 // We need to enforce the calling convention for the callsite, so that
7851 // argument ordering is enforced correctly, and that register allocation can
7852 // see that some registers may be assumed clobbered and have to preserve
7853 // them across calls to the intrinsic.
7854 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
7855 sdl, NodeTys, Ops);
7856 SDValue patchableNode = SDValue(MN, 0);
7857 DAG.setRoot(patchableNode);
7858 setValue(&I, patchableNode);
7859 return;
7860 }
7861 case Intrinsic::xray_typedevent: {
7862 // Here we want to make sure that the intrinsic behaves as if it has a
7863 // specific calling convention.
7864 const auto &Triple = DAG.getTarget().getTargetTriple();
7865 if (!Triple.isAArch64(64) && Triple.getArch() != Triple::x86_64)
7866 return;
7867
7869
7870 // We want to say that we always want the arguments in registers.
7871 // It's unclear to me how manipulating the selection DAG here forces callers
7872 // to provide arguments in registers instead of on the stack.
7873 SDValue LogTypeId = getValue(I.getArgOperand(0));
7874 SDValue LogEntryVal = getValue(I.getArgOperand(1));
7875 SDValue StrSizeVal = getValue(I.getArgOperand(2));
7876 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
7877 SDValue Chain = getRoot();
7878 Ops.push_back(LogTypeId);
7879 Ops.push_back(LogEntryVal);
7880 Ops.push_back(StrSizeVal);
7881 Ops.push_back(Chain);
7882
7883 // We need to enforce the calling convention for the callsite, so that
7884 // argument ordering is enforced correctly, and that register allocation can
7885 // see that some registers may be assumed clobbered and have to preserve
7886 // them across calls to the intrinsic.
7888 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops);
7889 SDValue patchableNode = SDValue(MN, 0);
7890 DAG.setRoot(patchableNode);
7891 setValue(&I, patchableNode);
7892 return;
7893 }
7894 case Intrinsic::experimental_deoptimize:
7896 return;
7897 case Intrinsic::stepvector:
7898 visitStepVector(I);
7899 return;
7900 case Intrinsic::vector_reduce_fadd:
7901 case Intrinsic::vector_reduce_fmul:
7902 case Intrinsic::vector_reduce_add:
7903 case Intrinsic::vector_reduce_mul:
7904 case Intrinsic::vector_reduce_and:
7905 case Intrinsic::vector_reduce_or:
7906 case Intrinsic::vector_reduce_xor:
7907 case Intrinsic::vector_reduce_smax:
7908 case Intrinsic::vector_reduce_smin:
7909 case Intrinsic::vector_reduce_umax:
7910 case Intrinsic::vector_reduce_umin:
7911 case Intrinsic::vector_reduce_fmax:
7912 case Intrinsic::vector_reduce_fmin:
7913 case Intrinsic::vector_reduce_fmaximum:
7914 case Intrinsic::vector_reduce_fminimum:
7915 visitVectorReduce(I, Intrinsic);
7916 return;
7917
7918 case Intrinsic::icall_branch_funnel: {
7920 Ops.push_back(getValue(I.getArgOperand(0)));
7921
7922 int64_t Offset;
7923 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7924 I.getArgOperand(1), Offset, DAG.getDataLayout()));
7925 if (!Base)
7927 "llvm.icall.branch.funnel operand must be a GlobalValue");
7928 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0));
7929
7930 struct BranchFunnelTarget {
7931 int64_t Offset;
7933 };
7935
7936 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) {
7937 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
7938 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
7939 if (ElemBase != Base)
7940 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
7941 "to the same GlobalValue");
7942
7943 SDValue Val = getValue(I.getArgOperand(Op + 1));
7944 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
7945 if (!GA)
7947 "llvm.icall.branch.funnel operand must be a GlobalValue");
7949 GA->getGlobal(), sdl, Val.getValueType(),
7950 GA->getOffset())});
7951 }
7952 llvm::sort(Targets,
7953 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
7954 return T1.Offset < T2.Offset;
7955 });
7956
7957 for (auto &T : Targets) {
7958 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32));
7959 Ops.push_back(T.Target);
7960 }
7961
7962 Ops.push_back(DAG.getRoot()); // Chain
7963 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl,
7964 MVT::Other, Ops),
7965 0);
7966 DAG.setRoot(N);
7967 setValue(&I, N);
7968 HasTailCall = true;
7969 return;
7970 }
7971
7972 case Intrinsic::wasm_landingpad_index:
7973 // Information this intrinsic contained has been transferred to
7974 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely
7975 // delete it now.
7976 return;
7977
7978 case Intrinsic::aarch64_settag:
7979 case Intrinsic::aarch64_settag_zero: {
7981 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero;
7983 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)),
7984 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)),
7985 ZeroMemory);
7986 DAG.setRoot(Val);
7987 setValue(&I, Val);
7988 return;
7989 }
7990 case Intrinsic::amdgcn_cs_chain: {
7991 assert(I.arg_size() == 5 && "Additional args not supported yet");
7992 assert(cast<ConstantInt>(I.getOperand(4))->isZero() &&
7993 "Non-zero flags not supported yet");
7994
7995 // At this point we don't care if it's amdgpu_cs_chain or
7996 // amdgpu_cs_chain_preserve.
7998
7999 Type *RetTy = I.getType();
8000 assert(RetTy->isVoidTy() && "Should not return");
8001
8002 SDValue Callee = getValue(I.getOperand(0));
8003
8004 // We only have 2 actual args: one for the SGPRs and one for the VGPRs.
8005 // We'll also tack the value of the EXEC mask at the end.
8007 Args.reserve(3);
8008
8009 for (unsigned Idx : {2, 3, 1}) {
8011 Arg.Node = getValue(I.getOperand(Idx));
8012 Arg.Ty = I.getOperand(Idx)->getType();
8013 Arg.setAttributes(&I, Idx);
8014 Args.push_back(Arg);
8015 }
8016
8017 assert(Args[0].IsInReg && "SGPR args should be marked inreg");
8018 assert(!Args[1].IsInReg && "VGPR args should not be marked inreg");
8019 Args[2].IsInReg = true; // EXEC should be inreg
8020
8022 CLI.setDebugLoc(getCurSDLoc())
8023 .setChain(getRoot())
8024 .setCallee(CC, RetTy, Callee, std::move(Args))
8025 .setNoReturn(true)
8026 .setTailCall(true)
8027 .setConvergent(I.isConvergent());
8028 CLI.CB = &I;
8029 std::pair<SDValue, SDValue> Result =
8030 lowerInvokable(CLI, /*EHPadBB*/ nullptr);
8031 (void)Result;
8032 assert(!Result.first.getNode() && !Result.second.getNode() &&
8033 "Should've lowered as tail call");
8034
8035 HasTailCall = true;
8036 return;
8037 }
8038 case Intrinsic::ptrmask: {
8039 SDValue Ptr = getValue(I.getOperand(0));
8040 SDValue Mask = getValue(I.getOperand(1));
8041
8042 // On arm64_32, pointers are 32 bits when stored in memory, but
8043 // zero-extended to 64 bits when in registers. Thus the mask is 32 bits to
8044 // match the index type, but the pointer is 64 bits, so the the mask must be
8045 // zero-extended up to 64 bits to match the pointer.
8046 EVT PtrVT =
8047 TLI.getValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8048 EVT MemVT =
8049 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
8050 assert(PtrVT == Ptr.getValueType());
8051 assert(MemVT == Mask.getValueType());
8052 if (MemVT != PtrVT)
8053 Mask = DAG.getPtrExtOrTrunc(Mask, sdl, PtrVT);
8054
8055 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, Mask));
8056 return;
8057 }
8058 case Intrinsic::threadlocal_address: {
8059 setValue(&I, getValue(I.getOperand(0)));
8060 return;
8061 }
8062 case Intrinsic::get_active_lane_mask: {
8063 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8064 SDValue Index = getValue(I.getOperand(0));
8065 EVT ElementVT = Index.getValueType();
8066
8067 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) {
8068 visitTargetIntrinsic(I, Intrinsic);
8069 return;
8070 }
8071
8072 SDValue TripCount = getValue(I.getOperand(1));
8073 EVT VecTy = EVT::getVectorVT(*DAG.getContext(), ElementVT,
8074 CCVT.getVectorElementCount());
8075
8076 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index);
8077 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount);
8078 SDValue VectorStep = DAG.getStepVector(sdl, VecTy);
8079 SDValue VectorInduction = DAG.getNode(
8080 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep);
8081 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction,
8082 VectorTripCount, ISD::CondCode::SETULT);
8083 setValue(&I, SetCC);
8084 return;
8085 }
8086 case Intrinsic::experimental_get_vector_length: {
8087 assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
8088 "Expected positive VF");
8089 unsigned VF = cast<ConstantInt>(I.getOperand(1))->getZExtValue();
8090 bool IsScalable = cast<ConstantInt>(I.getOperand(2))->isOne();
8091
8092 SDValue Count = getValue(I.getOperand(0));
8093 EVT CountVT = Count.getValueType();
8094
8095 if (!TLI.shouldExpandGetVectorLength(CountVT, VF, IsScalable)) {
8096 visitTargetIntrinsic(I, Intrinsic);
8097 return;
8098 }
8099
8100 // Expand to a umin between the trip count and the maximum elements the type
8101 // can hold.
8102 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8103
8104 // Extend the trip count to at least the result VT.
8105 if (CountVT.bitsLT(VT)) {
8106 Count = DAG.getNode(ISD::ZERO_EXTEND, sdl, VT, Count);
8107 CountVT = VT;
8108 }
8109
8110 SDValue MaxEVL = DAG.getElementCount(sdl, CountVT,
8111 ElementCount::get(VF, IsScalable));
8112
8113 SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL);
8114 // Clip to the result type if needed.
8115 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, sdl, VT, UMin);
8116
8117 setValue(&I, Trunc);
8118 return;
8119 }
8120 case Intrinsic::experimental_vector_partial_reduce_add: {
8121
8122 if (!TLI.shouldExpandPartialReductionIntrinsic(cast<IntrinsicInst>(&I))) {
8123 visitTargetIntrinsic(I, Intrinsic);
8124 return;
8125 }
8126
8127 setValue(&I, DAG.getPartialReduceAdd(sdl, EVT::getEVT(I.getType()),
8128 getValue(I.getOperand(0)),
8129 getValue(I.getOperand(1))));
8130 return;
8131 }
8132 case Intrinsic::experimental_cttz_elts: {
8133 auto DL = getCurSDLoc();
8134 SDValue Op = getValue(I.getOperand(0));
8135 EVT OpVT = Op.getValueType();
8136
8137 if (!TLI.shouldExpandCttzElements(OpVT)) {
8138 visitTargetIntrinsic(I, Intrinsic);
8139 return;
8140 }
8141
8142 if (OpVT.getScalarType() != MVT::i1) {
8143 // Compare the input vector elements to zero & use to count trailing zeros
8144 SDValue AllZero = DAG.getConstant(0, DL, OpVT);
8145 OpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
8146 OpVT.getVectorElementCount());
8147 Op = DAG.getSetCC(DL, OpVT, Op, AllZero, ISD::SETNE);
8148 }
8149
8150 // If the zero-is-poison flag is set, we can assume the upper limit
8151 // of the result is VF-1.
8152 bool ZeroIsPoison =
8153 !cast<ConstantSDNode>(getValue(I.getOperand(1)))->isZero();
8154 ConstantRange VScaleRange(1, true); // Dummy value.
8155 if (isa<ScalableVectorType>(I.getOperand(0)->getType()))
8156 VScaleRange = getVScaleRange(I.getCaller(), 64);
8157 unsigned EltWidth = TLI.getBitWidthForCttzElements(
8158 I.getType(), OpVT.getVectorElementCount(), ZeroIsPoison, &VScaleRange);
8159
8160 MVT NewEltTy = MVT::getIntegerVT(EltWidth);
8161
8162 // Create the new vector type & get the vector length
8163 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), NewEltTy,
8164 OpVT.getVectorElementCount());
8165
8166 SDValue VL =
8167 DAG.getElementCount(DL, NewEltTy, OpVT.getVectorElementCount());
8168
8169 SDValue StepVec = DAG.getStepVector(DL, NewVT);
8170 SDValue SplatVL = DAG.getSplat(NewVT, DL, VL);
8171 SDValue StepVL = DAG.getNode(ISD::SUB, DL, NewVT, SplatVL, StepVec);
8173 SDValue And = DAG.getNode(ISD::AND, DL, NewVT, StepVL, Ext);
8175 SDValue Sub = DAG.getNode(ISD::SUB, DL, NewEltTy, VL, Max);
8176
8177 EVT RetTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
8179
8180 setValue(&I, Ret);
8181 return;
8182 }
8183 case Intrinsic::vector_insert: {
8184 SDValue Vec = getValue(I.getOperand(0));
8185 SDValue SubVec = getValue(I.getOperand(1));
8186 SDValue Index = getValue(I.getOperand(2));
8187
8188 // The intrinsic's index type is i64, but the SDNode requires an index type
8189 // suitable for the target. Convert the index as required.
8190 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8191 if (Index.getValueType() != VectorIdxTy)
8192 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8193
8194 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8195 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec,
8196 Index));
8197 return;
8198 }
8199 case Intrinsic::vector_extract: {
8200 SDValue Vec = getValue(I.getOperand(0));
8201 SDValue Index = getValue(I.getOperand(1));
8202 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8203
8204 // The intrinsic's index type is i64, but the SDNode requires an index type
8205 // suitable for the target. Convert the index as required.
8206 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
8207 if (Index.getValueType() != VectorIdxTy)
8208 Index = DAG.getVectorIdxConstant(Index->getAsZExtVal(), sdl);
8209
8210 setValue(&I,
8211 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index));
8212 return;
8213 }
8214 case Intrinsic::experimental_vector_match: {
8215 SDValue Op1 = getValue(I.getOperand(0));
8216 SDValue Op2 = getValue(I.getOperand(1));
8217 SDValue Mask = getValue(I.getOperand(2));
8218 EVT Op1VT = Op1.getValueType();
8219 EVT Op2VT = Op2.getValueType();
8220 EVT ResVT = Mask.getValueType();
8221 unsigned SearchSize = Op2VT.getVectorNumElements();
8222
8223 // If the target has native support for this vector match operation, lower
8224 // the intrinsic untouched; otherwise, expand it below.
8225 if (!TLI.shouldExpandVectorMatch(Op1VT, SearchSize)) {
8226 visitTargetIntrinsic(I, Intrinsic);
8227 return;
8228 }
8229
8230 SDValue Ret = DAG.getConstant(0, sdl, ResVT);
8231
8232 for (unsigned i = 0; i < SearchSize; ++i) {
8234 Op2VT.getVectorElementType(), Op2,
8235 DAG.getVectorIdxConstant(i, sdl));
8236 SDValue Splat = DAG.getNode(ISD::SPLAT_VECTOR, sdl, Op1VT, Op2Elem);
8237 SDValue Cmp = DAG.getSetCC(sdl, ResVT, Op1, Splat, ISD::SETEQ);
8238 Ret = DAG.getNode(ISD::OR, sdl, ResVT, Ret, Cmp);
8239 }
8240
8241 setValue(&I, DAG.getNode(ISD::AND, sdl, ResVT, Ret, Mask));
8242 return;
8243 }
8244 case Intrinsic::vector_reverse:
8245 visitVectorReverse(I);
8246 return;
8247 case Intrinsic::vector_splice:
8248 visitVectorSplice(I);
8249 return;
8250 case Intrinsic::callbr_landingpad:
8251 visitCallBrLandingPad(I);
8252 return;
8253 case Intrinsic::vector_interleave2:
8254 visitVectorInterleave(I);
8255 return;
8256 case Intrinsic::vector_deinterleave2:
8257 visitVectorDeinterleave(I);
8258 return;
8259 case Intrinsic::experimental_vector_compress:
8261 getValue(I.getArgOperand(0)).getValueType(),
8262 getValue(I.getArgOperand(0)),
8263 getValue(I.getArgOperand(1)),
8264 getValue(I.getArgOperand(2)), Flags));
8265 return;
8266 case Intrinsic::experimental_convergence_anchor:
8267 case Intrinsic::experimental_convergence_entry:
8268 case Intrinsic::experimental_convergence_loop:
8269 visitConvergenceControl(I, Intrinsic);
8270 return;
8271 case Intrinsic::experimental_vector_histogram_add: {
8272 visitVectorHistogram(I, Intrinsic);
8273 return;
8274 }
8275 case Intrinsic::experimental_vector_extract_last_active: {
8276 visitVectorExtractLastActive(I, Intrinsic);
8277 return;
8278 }
8279 }
8280}
8281
8282void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
8283 const ConstrainedFPIntrinsic &FPI) {
8284 SDLoc sdl = getCurSDLoc();
8285
8286 // We do not need to serialize constrained FP intrinsics against
8287 // each other or against (nonvolatile) loads, so they can be
8288 // chained like loads.
8289 SDValue Chain = DAG.getRoot();
8291 Opers.push_back(Chain);
8292 for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)
8293 Opers.push_back(getValue(FPI.getArgOperand(I)));
8294
8295 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) {
8296 assert(Result.getNode()->getNumValues() == 2);
8297
8298 // Push node to the appropriate list so that future instructions can be
8299 // chained up correctly.
8300 SDValue OutChain = Result.getValue(1);
8301 switch (EB) {
8303 // The only reason why ebIgnore nodes still need to be chained is that
8304 // they might depend on the current rounding mode, and therefore must
8305 // not be moved across instruction that may change that mode.
8306 [[fallthrough]];
8308 // These must not be moved across calls or instructions that may change
8309 // floating-point exception masks.
8310 PendingConstrainedFP.push_back(OutChain);
8311 break;
8313 // These must not be moved across calls or instructions that may change
8314 // floating-point exception masks or read floating-point exception flags.
8315 // In addition, they cannot be optimized out even if unused.
8316 PendingConstrainedFPStrict.push_back(OutChain);
8317 break;
8318 }
8319 };
8320
8322 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType());
8323 SDVTList VTs = DAG.getVTList(VT, MVT::Other);
8325
8328 Flags.setNoFPExcept(true);
8329
8330 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI))
8331 Flags.copyFMF(*FPOp);
8332
8333 unsigned Opcode;
8334 switch (FPI.getIntrinsicID()) {
8335 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
8336#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
8337 case Intrinsic::INTRINSIC: \
8338 Opcode = ISD::STRICT_##DAGN; \
8339 break;
8340#include "llvm/IR/ConstrainedOps.def"
8341 case Intrinsic::experimental_constrained_fmuladd: {
8342 Opcode = ISD::STRICT_FMA;
8343 // Break fmuladd into fmul and fadd.
8346 Opers.pop_back();
8347 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags);
8348 pushOutChain(Mul, EB);
8349 Opcode = ISD::STRICT_FADD;
8350 Opers.clear();
8351 Opers.push_back(Mul.getValue(1));
8352 Opers.push_back(Mul.getValue(0));
8353 Opers.push_back(getValue(FPI.getArgOperand(2)));
8354 }
8355 break;
8356 }
8357 }
8358
8359 // A few strict DAG nodes carry additional operands that are not
8360 // set up by the default code above.
8361 switch (Opcode) {
8362 default: break;
8364 Opers.push_back(
8366 break;
8367 case ISD::STRICT_FSETCC:
8368 case ISD::STRICT_FSETCCS: {
8369 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI);
8370 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate());
8371 if (TM.Options.NoNaNsFPMath)
8372 Condition = getFCmpCodeWithoutNaN(Condition);
8373 Opers.push_back(DAG.getCondCode(Condition));
8374 break;
8375 }
8376 }
8377
8378 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags);
8379 pushOutChain(Result, EB);
8380
8381 SDValue FPResult = Result.getValue(0);
8382 setValue(&FPI, FPResult);
8383}
8384
8385static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) {
8386 std::optional<unsigned> ResOPC;
8387 switch (VPIntrin.getIntrinsicID()) {
8388 case Intrinsic::vp_ctlz: {
8389 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8390 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ;
8391 break;
8392 }
8393 case Intrinsic::vp_cttz: {
8394 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8395 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ;
8396 break;
8397 }
8398 case Intrinsic::vp_cttz_elts: {
8399 bool IsZeroPoison = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne();
8400 ResOPC = IsZeroPoison ? ISD::VP_CTTZ_ELTS_ZERO_UNDEF : ISD::VP_CTTZ_ELTS;
8401 break;
8402 }
8403#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \
8404 case Intrinsic::VPID: \
8405 ResOPC = ISD::VPSD; \
8406 break;
8407#include "llvm/IR/VPIntrinsics.def"
8408 }
8409
8410 if (!ResOPC)
8412 "Inconsistency: no SDNode available for this VPIntrinsic!");
8413
8414 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD ||
8415 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) {
8416 if (VPIntrin.getFastMathFlags().allowReassoc())
8417 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD
8418 : ISD::VP_REDUCE_FMUL;
8419 }
8420
8421 return *ResOPC;
8422}
8423
8424void SelectionDAGBuilder::visitVPLoad(
8425 const VPIntrinsic &VPIntrin, EVT VT,
8426 const SmallVectorImpl<SDValue> &OpValues) {
8427 SDLoc DL = getCurSDLoc();
8428 Value *PtrOperand = VPIntrin.getArgOperand(0);
8429 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8430 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8431 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8432 SDValue LD;
8433 // Do not serialize variable-length loads of constant memory with
8434 // anything.
8435 if (!Alignment)
8436 Alignment = DAG.getEVTAlign(VT);
8437 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8438 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8439 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8442 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8443 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
8444 MMO, false /*IsExpanding */);
8445 if (AddToChain)
8446 PendingLoads.push_back(LD.getValue(1));
8447 setValue(&VPIntrin, LD);
8448}
8449
8450void SelectionDAGBuilder::visitVPGather(
8451 const VPIntrinsic &VPIntrin, EVT VT,
8452 const SmallVectorImpl<SDValue> &OpValues) {
8453 SDLoc DL = getCurSDLoc();
8455 Value *PtrOperand = VPIntrin.getArgOperand(0);
8456 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8457 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8458 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8459 SDValue LD;
8460 if (!Alignment)
8461 Alignment = DAG.getEVTAlign(VT.getScalarType());
8462 unsigned AS =
8463 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8466 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8467 SDValue Base, Index, Scale;
8468 ISD::MemIndexType IndexType;
8469 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8470 this, VPIntrin.getParent(),
8471 VT.getScalarStoreSize());
8472 if (!UniformBase) {
8474 Index = getValue(PtrOperand);
8475 IndexType = ISD::SIGNED_SCALED;
8477 }
8478 EVT IdxVT = Index.getValueType();
8479 EVT EltTy = IdxVT.getVectorElementType();
8480 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8481 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8482 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8483 }
8484 LD = DAG.getGatherVP(
8485 DAG.getVTList(VT, MVT::Other), VT, DL,
8486 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
8487 IndexType);
8488 PendingLoads.push_back(LD.getValue(1));
8489 setValue(&VPIntrin, LD);
8490}
8491
8492void SelectionDAGBuilder::visitVPStore(
8493 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8494 SDLoc DL = getCurSDLoc();
8495 Value *PtrOperand = VPIntrin.getArgOperand(1);
8496 EVT VT = OpValues[0].getValueType();
8497 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8498 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8499 SDValue ST;
8500 if (!Alignment)
8501 Alignment = DAG.getEVTAlign(VT);
8502 SDValue Ptr = OpValues[1];
8503 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
8506 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8507 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
8508 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
8509 /* IsTruncating */ false, /*IsCompressing*/ false);
8510 DAG.setRoot(ST);
8511 setValue(&VPIntrin, ST);
8512}
8513
8514void SelectionDAGBuilder::visitVPScatter(
8515 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8516 SDLoc DL = getCurSDLoc();
8518 Value *PtrOperand = VPIntrin.getArgOperand(1);
8519 EVT VT = OpValues[0].getValueType();
8520 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8521 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8522 SDValue ST;
8523 if (!Alignment)
8524 Alignment = DAG.getEVTAlign(VT.getScalarType());
8525 unsigned AS =
8526 PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8529 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8530 SDValue Base, Index, Scale;
8531 ISD::MemIndexType IndexType;
8532 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
8533 this, VPIntrin.getParent(),
8534 VT.getScalarStoreSize());
8535 if (!UniformBase) {
8537 Index = getValue(PtrOperand);
8538 IndexType = ISD::SIGNED_SCALED;
8539 Scale =
8541 }
8542 EVT IdxVT = Index.getValueType();
8543 EVT EltTy = IdxVT.getVectorElementType();
8544 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) {
8545 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy);
8546 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index);
8547 }
8548 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
8549 {getMemoryRoot(), OpValues[0], Base, Index, Scale,
8550 OpValues[2], OpValues[3]},
8551 MMO, IndexType);
8552 DAG.setRoot(ST);
8553 setValue(&VPIntrin, ST);
8554}
8555
8556void SelectionDAGBuilder::visitVPStridedLoad(
8557 const VPIntrinsic &VPIntrin, EVT VT,
8558 const SmallVectorImpl<SDValue> &OpValues) {
8559 SDLoc DL = getCurSDLoc();
8560 Value *PtrOperand = VPIntrin.getArgOperand(0);
8561 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8562 if (!Alignment)
8563 Alignment = DAG.getEVTAlign(VT.getScalarType());
8564 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8565 const MDNode *Ranges = getRangeMetadata(VPIntrin);
8566 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
8567 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML);
8568 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8569 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8572 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8573
8574 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
8575 OpValues[2], OpValues[3], MMO,
8576 false /*IsExpanding*/);
8577
8578 if (AddToChain)
8579 PendingLoads.push_back(LD.getValue(1));
8580 setValue(&VPIntrin, LD);
8581}
8582
8583void SelectionDAGBuilder::visitVPStridedStore(
8584 const VPIntrinsic &VPIntrin, const SmallVectorImpl<SDValue> &OpValues) {
8585 SDLoc DL = getCurSDLoc();
8586 Value *PtrOperand = VPIntrin.getArgOperand(1);
8587 EVT VT = OpValues[0].getValueType();
8588 MaybeAlign Alignment = VPIntrin.getPointerAlignment();
8589 if (!Alignment)
8590 Alignment = DAG.getEVTAlign(VT.getScalarType());
8591 AAMDNodes AAInfo = VPIntrin.getAAMetadata();
8592 unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8595 LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8596
8598 getMemoryRoot(), DL, OpValues[0], OpValues[1],
8599 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3],
8600 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false,
8601 /*IsCompressing*/ false);
8602
8603 DAG.setRoot(ST);
8604 setValue(&VPIntrin, ST);
8605}
8606
8607void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) {
8609 SDLoc DL = getCurSDLoc();
8610
8611 ISD::CondCode Condition;
8613 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy();
8614 if (IsFP) {
8615 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan)
8616 // flags, but calls that don't return floating-point types can't be
8617 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too.
8618 Condition = getFCmpCondCode(CondCode);
8619 if (TM.Options.NoNaNsFPMath)
8620 Condition = getFCmpCodeWithoutNaN(Condition);
8621 } else {
8622 Condition = getICmpCondCode(CondCode);
8623 }
8624
8625 SDValue Op1 = getValue(VPIntrin.getOperand(0));
8626 SDValue Op2 = getValue(VPIntrin.getOperand(1));
8627 // #2 is the condition code
8628 SDValue MaskOp = getValue(VPIntrin.getOperand(3));
8629 SDValue EVL = getValue(VPIntrin.getOperand(4));
8630 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8631 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8632 "Unexpected target EVL type");
8633 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL);
8634
8636 VPIntrin.getType());
8637 setValue(&VPIntrin,
8638 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL));
8639}
8640
8641void SelectionDAGBuilder::visitVectorPredicationIntrinsic(
8642 const VPIntrinsic &VPIntrin) {
8643 SDLoc DL = getCurSDLoc();
8644 unsigned Opcode = getISDForVPIntrinsic(VPIntrin);
8645
8646 auto IID = VPIntrin.getIntrinsicID();
8647
8648 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin))
8649 return visitVPCmp(*CmpI);
8650
8651 SmallVector<EVT, 4> ValueVTs;
8653 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs);
8654 SDVTList VTs = DAG.getVTList(ValueVTs);
8655
8656 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID);
8657
8658 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy();
8659 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) &&
8660 "Unexpected target EVL type");
8661
8662 // Request operands.
8663 SmallVector<SDValue, 7> OpValues;
8664 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
8665 auto Op = getValue(VPIntrin.getArgOperand(I));
8666 if (I == EVLParamPos)
8667 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
8668 OpValues.push_back(Op);
8669 }
8670
8671 switch (Opcode) {
8672 default: {
8673 SDNodeFlags SDFlags;
8674 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8675 SDFlags.copyFMF(*FPMO);
8676 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags);
8677 setValue(&VPIntrin, Result);
8678 break;
8679 }
8680 case ISD::VP_LOAD:
8681 visitVPLoad(VPIntrin, ValueVTs[0], OpValues);
8682 break;
8683 case ISD::VP_GATHER:
8684 visitVPGather(VPIntrin, ValueVTs[0], OpValues);
8685 break;
8686 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
8687 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues);
8688 break;
8689 case ISD::VP_STORE:
8690 visitVPStore(VPIntrin, OpValues);
8691 break;
8692 case ISD::VP_SCATTER:
8693 visitVPScatter(VPIntrin, OpValues);
8694 break;
8695 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
8696 visitVPStridedStore(VPIntrin, OpValues);
8697 break;
8698 case ISD::VP_FMULADD: {
8699 assert(OpValues.size() == 5 && "Unexpected number of operands");
8700 SDNodeFlags SDFlags;
8701 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin))
8702 SDFlags.copyFMF(*FPMO);
8705 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags));
8706 } else {
8708 ISD::VP_FMUL, DL, VTs,
8709 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags);
8710 SDValue Add =
8711 DAG.getNode(ISD::VP_FADD, DL, VTs,
8712 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags);
8713 setValue(&VPIntrin, Add);
8714 }
8715 break;
8716 }
8717 case ISD::VP_IS_FPCLASS: {
8718 const DataLayout DLayout = DAG.getDataLayout();
8719 EVT DestVT = TLI.getValueType(DLayout, VPIntrin.getType());
8720 auto Constant = OpValues[1]->getAsZExtVal();
8722 SDValue V = DAG.getNode(ISD::VP_IS_FPCLASS, DL, DestVT,
8723 {OpValues[0], Check, OpValues[2], OpValues[3]});
8724 setValue(&VPIntrin, V);
8725 return;
8726 }
8727 case ISD::VP_INTTOPTR: {
8728 SDValue N = OpValues[0];
8729 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType());
8730 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType());
8731 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8732 OpValues[2]);
8733 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8734 OpValues[2]);
8735 setValue(&VPIntrin, N);
8736 break;
8737 }
8738 case ISD::VP_PTRTOINT: {
8739 SDValue N = OpValues[0];
8741 VPIntrin.getType());
8742 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(),
8743 VPIntrin.getOperand(0)->getType());
8744 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1],
8745 OpValues[2]);
8746 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1],
8747 OpValues[2]);
8748 setValue(&VPIntrin, N);
8749 break;
8750 }
8751 case ISD::VP_ABS:
8752 case ISD::VP_CTLZ:
8753 case ISD::VP_CTLZ_ZERO_UNDEF:
8754 case ISD::VP_CTTZ:
8755 case ISD::VP_CTTZ_ZERO_UNDEF:
8756 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
8757 case ISD::VP_CTTZ_ELTS: {
8758 SDValue Result =
8759 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]});
8760 setValue(&VPIntrin, Result);
8761 break;
8762 }
8763 }
8764}
8765
8766SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain,
8767 const BasicBlock *EHPadBB,
8768 MCSymbol *&BeginLabel) {
8770
8771 // Insert a label before the invoke call to mark the try range. This can be
8772 // used to detect deletion of the invoke via the MachineModuleInfo.
8773 BeginLabel = MF.getContext().createTempSymbol();
8774
8775 // For SjLj, keep track of which landing pads go with which invokes
8776 // so as to maintain the ordering of pads in the LSDA.
8777 unsigned CallSiteIndex = FuncInfo.getCurrentCallSite();
8778 if (CallSiteIndex) {
8779 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
8780 LPadToCallSiteMap[FuncInfo.getMBB(EHPadBB)].push_back(CallSiteIndex);
8781
8782 // Now that the call site is handled, stop tracking it.
8784 }
8785
8786 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel);
8787}
8788
8789SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II,
8790 const BasicBlock *EHPadBB,
8791 MCSymbol *BeginLabel) {
8792 assert(BeginLabel && "BeginLabel should've been set");
8793
8795
8796 // Insert a label at the end of the invoke call to mark the try range. This
8797 // can be used to detect deletion of the invoke via the MachineModuleInfo.
8798 MCSymbol *EndLabel = MF.getContext().createTempSymbol();
8799 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel);
8800
8801 // Inform MachineModuleInfo of range.
8803 // There is a platform (e.g. wasm) that uses funclet style IR but does not
8804 // actually use outlined funclets and their LSDA info style.
8805 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
8806 assert(II && "II should've been set");
8807 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
8808 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel);
8809 } else if (!isScopedEHPersonality(Pers)) {
8810 assert(EHPadBB);
8811 MF.addInvoke(FuncInfo.getMBB(EHPadBB), BeginLabel, EndLabel);
8812 }
8813
8814 return Chain;
8815}
8816
8817std::pair<SDValue, SDValue>
8819 const BasicBlock *EHPadBB) {
8820 MCSymbol *BeginLabel = nullptr;
8821
8822 if (EHPadBB) {
8823 // Both PendingLoads and PendingExports must be flushed here;
8824 // this call might not return.
8825 (void)getRoot();
8826 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel));
8827 CLI.setChain(getRoot());
8828 }
8829
8831 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
8832
8833 assert((CLI.IsTailCall || Result.second.getNode()) &&
8834 "Non-null chain expected with non-tail call!");
8835 assert((Result.second.getNode() || !Result.first.getNode()) &&
8836 "Null value expected with tail call!");
8837
8838 if (!Result.second.getNode()) {
8839 // As a special case, a null chain means that a tail call has been emitted
8840 // and the DAG root is already updated.
8841 HasTailCall = true;
8842
8843 // Since there's no actual continuation from this block, nothing can be
8844 // relying on us setting vregs for them.
8845 PendingExports.clear();
8846 } else {
8847 DAG.setRoot(Result.second);
8848 }
8849
8850 if (EHPadBB) {
8851 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB,
8852 BeginLabel));
8853 Result.second = getRoot();
8854 }
8855
8856 return Result;
8857}
8858
8860 bool isTailCall, bool isMustTailCall,
8861 const BasicBlock *EHPadBB,
8862 const TargetLowering::PtrAuthInfo *PAI) {
8863 auto &DL = DAG.getDataLayout();
8864 FunctionType *FTy = CB.getFunctionType();
8865 Type *RetTy = CB.getType();
8866
8868 Args.reserve(CB.arg_size());
8869
8870 const Value *SwiftErrorVal = nullptr;
8872
8873 if (isTailCall) {
8874 // Avoid emitting tail calls in functions with the disable-tail-calls
8875 // attribute.
8876 auto *Caller = CB.getParent()->getParent();
8877 if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() ==
8878 "true" && !isMustTailCall)
8879 isTailCall = false;
8880
8881 // We can't tail call inside a function with a swifterror argument. Lowering
8882 // does not support this yet. It would have to move into the swifterror
8883 // register before the call.
8884 if (TLI.supportSwiftError() &&
8885 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
8886 isTailCall = false;
8887 }
8888
8889 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
8891 const Value *V = *I;
8892
8893 // Skip empty types
8894 if (V->getType()->isEmptyTy())
8895 continue;
8896
8897 SDValue ArgNode = getValue(V);
8898 Entry.Node = ArgNode; Entry.Ty = V->getType();
8899
8900 Entry.setAttributes(&CB, I - CB.arg_begin());
8901
8902 // Use swifterror virtual register as input to the call.
8903 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
8904 SwiftErrorVal = V;
8905 // We find the virtual register for the actual swifterror argument.
8906 // Instead of using the Value, we use the virtual register instead.
8907 Entry.Node =
8909 EVT(TLI.getPointerTy(DL)));
8910 }
8911
8912 Args.push_back(Entry);
8913
8914 // If we have an explicit sret argument that is an Instruction, (i.e., it
8915 // might point to function-local memory), we can't meaningfully tail-call.
8916 if (Entry.IsSRet && isa<Instruction>(V))
8917 isTailCall = false;
8918 }
8919
8920 // If call site has a cfguardtarget operand bundle, create and add an
8921 // additional ArgListEntry.
8922 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
8924 Value *V = Bundle->Inputs[0];
8925 SDValue ArgNode = getValue(V);
8926 Entry.Node = ArgNode;
8927 Entry.Ty = V->getType();
8928 Entry.IsCFGuardTarget = true;
8929 Args.push_back(Entry);
8930 }
8931
8932 // Check if target-independent constraints permit a tail call here.
8933 // Target-dependent constraints are checked within TLI->LowerCallTo.
8934 if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget()))
8935 isTailCall = false;
8936
8937 // Disable tail calls if there is an swifterror argument. Targets have not
8938 // been updated to support tail calls.
8939 if (TLI.supportSwiftError() && SwiftErrorVal)
8940 isTailCall = false;
8941
8942 ConstantInt *CFIType = nullptr;
8943 if (CB.isIndirectCall()) {
8944 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) {
8945 if (!TLI.supportKCFIBundles())
8947 "Target doesn't support calls with kcfi operand bundles.");
8948 CFIType = cast<ConstantInt>(Bundle->Inputs[0]);
8949 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type");
8950 }
8951 }
8952
8953 SDValue ConvControlToken;
8954 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
8955 auto *Token = Bundle->Inputs[0].get();
8956 ConvControlToken = getValue(Token);
8957 }
8958
8961 .setChain(getRoot())
8962 .setCallee(RetTy, FTy, Callee, std::move(Args), CB)
8963 .setTailCall(isTailCall)
8967 .setCFIType(CFIType)
8968 .setConvergenceControlToken(ConvControlToken);
8969
8970 // Set the pointer authentication info if we have it.
8971 if (PAI) {
8972 if (!TLI.supportPtrAuthBundles())
8974 "This target doesn't support calls with ptrauth operand bundles.");
8975 CLI.setPtrAuth(*PAI);
8976 }
8977
8978 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8979
8980 if (Result.first.getNode()) {
8981 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first);
8982 setValue(&CB, Result.first);
8983 }
8984
8985 // The last element of CLI.InVals has the SDValue for swifterror return.
8986 // Here we copy it to a virtual register and update SwiftErrorMap for
8987 // book-keeping.
8988 if (SwiftErrorVal && TLI.supportSwiftError()) {
8989 // Get the last element of InVals.
8990 SDValue Src = CLI.InVals.back();
8991 Register VReg =
8992 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal);
8993 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
8994 DAG.setRoot(CopyNode);
8995 }
8996}
8997
8998static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
8999 SelectionDAGBuilder &Builder) {
9000 // Check to see if this load can be trivially constant folded, e.g. if the
9001 // input is from a string literal.
9002 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
9003 // Cast pointer to the type we really want to load.
9004 Type *LoadTy =
9005 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
9006 if (LoadVT.isVector())
9007 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements());
9008
9009 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
9010 PointerType::getUnqual(LoadTy));
9011
9012 if (const Constant *LoadCst =
9013 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput),
9014 LoadTy, Builder.DAG.getDataLayout()))
9015 return Builder.getValue(LoadCst);
9016 }
9017
9018 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
9019 // still constant memory, the input chain can be the entry node.
9020 SDValue Root;
9021 bool ConstantMemory = false;
9022
9023 // Do not serialize (non-volatile) loads of constant memory with anything.
9024 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
9025 Root = Builder.DAG.getEntryNode();
9026 ConstantMemory = true;
9027 } else {
9028 // Do not serialize non-volatile loads against each other.
9029 Root = Builder.DAG.getRoot();
9030 }
9031
9032 SDValue Ptr = Builder.getValue(PtrVal);
9033 SDValue LoadVal =
9034 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr,
9035 MachinePointerInfo(PtrVal), Align(1));
9036
9037 if (!ConstantMemory)
9038 Builder.PendingLoads.push_back(LoadVal.getValue(1));
9039 return LoadVal;
9040}
9041
9042/// Record the value for an instruction that produces an integer result,
9043/// converting the type where necessary.
9044void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
9045 SDValue Value,
9046 bool IsSigned) {
9048 I.getType(), true);
9049 Value = DAG.getExtOrTrunc(IsSigned, Value, getCurSDLoc(), VT);
9050 setValue(&I, Value);
9051}
9052
9053/// See if we can lower a memcmp/bcmp call into an optimized form. If so, return
9054/// true and lower it. Otherwise return false, and it will be lowered like a
9055/// normal call.
9056/// The caller already checked that \p I calls the appropriate LibFunc with a
9057/// correct prototype.
9058bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
9059 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
9060 const Value *Size = I.getArgOperand(2);
9061 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size));
9062 if (CSize && CSize->getZExtValue() == 0) {
9064 I.getType(), true);
9065 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
9066 return true;
9067 }
9068
9070 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
9071 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
9073 if (Res.first.getNode()) {
9074 processIntegerCallValue(I, Res.first, true);
9075 PendingLoads.push_back(Res.second);
9076 return true;
9077 }
9078
9079 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
9080 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
9081 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
9082 return false;
9083
9084 // If the target has a fast compare for the given size, it will return a
9085 // preferred load type for that size. Require that the load VT is legal and
9086 // that the target supports unaligned loads of that type. Otherwise, return
9087 // INVALID.
9088 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
9090 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
9091 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
9092 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
9093 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
9094 // TODO: Check alignment of src and dest ptrs.
9095 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
9096 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
9097 if (!TLI.isTypeLegal(LVT) ||
9098 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
9099 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
9101 }
9102
9103 return LVT;
9104 };
9105
9106 // This turns into unaligned loads. We only do this if the target natively
9107 // supports the MVT we'll be loading or if it is small enough (<= 4) that
9108 // we'll only produce a small number of byte loads.
9109 MVT LoadVT;
9110 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
9111 switch (NumBitsToCompare) {
9112 default:
9113 return false;
9114 case 16:
9115 LoadVT = MVT::i16;
9116 break;
9117 case 32:
9118 LoadVT = MVT::i32;
9119 break;
9120 case 64:
9121 case 128:
9122 case 256:
9123 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
9124 break;
9125 }
9126
9127 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
9128 return false;
9129
9130 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
9131 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
9132
9133 // Bitcast to a wide integer type if the loads are vectors.
9134 if (LoadVT.isVector()) {
9135 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
9136 LoadL = DAG.getBitcast(CmpVT, LoadL);
9137 LoadR = DAG.getBitcast(CmpVT, LoadR);
9138 }
9139
9140 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
9141 processIntegerCallValue(I, Cmp, false);
9142 return true;
9143}
9144
9145/// See if we can lower a memchr call into an optimized form. If so, return
9146/// true and lower it. Otherwise return false, and it will be lowered like a
9147/// normal call.
9148/// The caller already checked that \p I calls the appropriate LibFunc with a
9149/// correct prototype.
9150bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
9151 const Value *Src = I.getArgOperand(0);
9152 const Value *Char = I.getArgOperand(1);
9153 const Value *Length = I.getArgOperand(2);
9154
9156 std::pair<SDValue, SDValue> Res =
9158 getValue(Src), getValue(Char), getValue(Length),
9159 MachinePointerInfo(Src));
9160 if (Res.first.getNode()) {
9161 setValue(&I, Res.first);
9162 PendingLoads.push_back(Res.second);
9163 return true;
9164 }
9165
9166 return false;
9167}
9168
9169/// See if we can lower a mempcpy call into an optimized form. If so, return
9170/// true and lower it. Otherwise return false, and it will be lowered like a
9171/// normal call.
9172/// The caller already checked that \p I calls the appropriate LibFunc with a
9173/// correct prototype.
9174bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
9175 SDValue Dst = getValue(I.getArgOperand(0));
9176 SDValue Src = getValue(I.getArgOperand(1));
9177 SDValue Size = getValue(I.getArgOperand(2));
9178
9179 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne();
9180 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne();
9181 // DAG::getMemcpy needs Alignment to be defined.
9182 Align Alignment = std::min(DstAlign, SrcAlign);
9183
9184 SDLoc sdl = getCurSDLoc();
9185
9186 // In the mempcpy context we need to pass in a false value for isTailCall
9187 // because the return pointer needs to be adjusted by the size of
9188 // the copied memory.
9189 SDValue Root = getMemoryRoot();
9190 SDValue MC = DAG.getMemcpy(
9191 Root, sdl, Dst, Src, Size, Alignment, false, false, /*CI=*/nullptr,
9192 std::nullopt, MachinePointerInfo(I.getArgOperand(0)),
9193 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata());
9194 assert(MC.getNode() != nullptr &&
9195 "** memcpy should not be lowered as TailCall in mempcpy context **");
9196 DAG.setRoot(MC);
9197
9198 // Check if Size needs to be truncated or extended.
9199 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
9200
9201 // Adjust return pointer to point just past the last dst byte.
9202 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
9203 Dst, Size);
9204 setValue(&I, DstPlusSize);
9205 return true;
9206}
9207
9208/// See if we can lower a strcpy call into an optimized form. If so, return
9209/// true and lower it, otherwise return false and it will be lowered like a
9210/// normal call.
9211/// The caller already checked that \p I calls the appropriate LibFunc with a
9212/// correct prototype.
9213bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
9214 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9215
9217 std::pair<SDValue, SDValue> Res =
9219 getValue(Arg0), getValue(Arg1),
9220 MachinePointerInfo(Arg0),
9221 MachinePointerInfo(Arg1), isStpcpy);
9222 if (Res.first.getNode()) {
9223 setValue(&I, Res.first);
9224 DAG.setRoot(Res.second);
9225 return true;
9226 }
9227
9228 return false;
9229}
9230
9231/// See if we can lower a strcmp call into an optimized form. If so, return
9232/// true and lower it, otherwise return false and it will be lowered like a
9233/// normal call.
9234/// The caller already checked that \p I calls the appropriate LibFunc with a
9235/// correct prototype.
9236bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
9237 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9238
9240 std::pair<SDValue, SDValue> Res =
9242 getValue(Arg0), getValue(Arg1),
9243 MachinePointerInfo(Arg0),
9244 MachinePointerInfo(Arg1));
9245 if (Res.first.getNode()) {
9246 processIntegerCallValue(I, Res.first, true);
9247 PendingLoads.push_back(Res.second);
9248 return true;
9249 }
9250
9251 return false;
9252}
9253
9254/// See if we can lower a strlen call into an optimized form. If so, return
9255/// true and lower it, otherwise return false and it will be lowered like a
9256/// normal call.
9257/// The caller already checked that \p I calls the appropriate LibFunc with a
9258/// correct prototype.
9259bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
9260 const Value *Arg0 = I.getArgOperand(0);
9261
9263 std::pair<SDValue, SDValue> Res =
9265 getValue(Arg0), MachinePointerInfo(Arg0));
9266 if (Res.first.getNode()) {
9267 processIntegerCallValue(I, Res.first, false);
9268 PendingLoads.push_back(Res.second);
9269 return true;
9270 }
9271
9272 return false;
9273}
9274
9275/// See if we can lower a strnlen call into an optimized form. If so, return
9276/// true and lower it, otherwise return false and it will be lowered like a
9277/// normal call.
9278/// The caller already checked that \p I calls the appropriate LibFunc with a
9279/// correct prototype.
9280bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
9281 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
9282
9284 std::pair<SDValue, SDValue> Res =
9286 getValue(Arg0), getValue(Arg1),
9287 MachinePointerInfo(Arg0));
9288 if (Res.first.getNode()) {
9289 processIntegerCallValue(I, Res.first, false);
9290 PendingLoads.push_back(Res.second);
9291 return true;
9292 }
9293
9294 return false;
9295}
9296
9297/// See if we can lower a unary floating-point operation into an SDNode with
9298/// the specified Opcode. If so, return true and lower it, otherwise return
9299/// false and it will be lowered like a normal call.
9300/// The caller already checked that \p I calls the appropriate LibFunc with a
9301/// correct prototype.
9302bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
9303 unsigned Opcode) {
9304 // We already checked this call's prototype; verify it doesn't modify errno.
9305 if (!I.onlyReadsMemory())
9306 return false;
9307
9309 Flags.copyFMF(cast<FPMathOperator>(I));
9310
9311 SDValue Tmp = getValue(I.getArgOperand(0));
9312 setValue(&I,
9313 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags));
9314 return true;
9315}
9316
9317/// See if we can lower a binary floating-point operation into an SDNode with
9318/// the specified Opcode. If so, return true and lower it. Otherwise return
9319/// false, and it will be lowered like a normal call.
9320/// The caller already checked that \p I calls the appropriate LibFunc with a
9321/// correct prototype.
9322bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
9323 unsigned Opcode) {
9324 // We already checked this call's prototype; verify it doesn't modify errno.
9325 if (!I.onlyReadsMemory())
9326 return false;
9327
9329 Flags.copyFMF(cast<FPMathOperator>(I));
9330
9331 SDValue Tmp0 = getValue(I.getArgOperand(0));
9332 SDValue Tmp1 = getValue(I.getArgOperand(1));
9333 EVT VT = Tmp0.getValueType();
9334 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags));
9335 return true;
9336}
9337
9338void SelectionDAGBuilder::visitCall(const CallInst &I) {
9339 // Handle inline assembly differently.
9340 if (I.isInlineAsm()) {
9341 visitInlineAsm(I);
9342 return;
9343 }
9344
9346
9347 if (Function *F = I.getCalledFunction()) {
9348 if (F->isDeclaration()) {
9349 // Is this an LLVM intrinsic or a target-specific intrinsic?
9350 unsigned IID = F->getIntrinsicID();
9351 if (!IID)
9352 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
9353 IID = II->getIntrinsicID(F);
9354
9355 if (IID) {
9356 visitIntrinsicCall(I, IID);
9357 return;
9358 }
9359 }
9360
9361 // Check for well-known libc/libm calls. If the function is internal, it
9362 // can't be a library call. Don't do the check if marked as nobuiltin for
9363 // some reason or the call site requires strict floating point semantics.
9364 LibFunc Func;
9365 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
9366 F->hasName() && LibInfo->getLibFunc(*F, Func) &&
9368 switch (Func) {
9369 default: break;
9370 case LibFunc_bcmp:
9371 if (visitMemCmpBCmpCall(I))
9372 return;
9373 break;
9374 case LibFunc_copysign:
9375 case LibFunc_copysignf:
9376 case LibFunc_copysignl:
9377 // We already checked this call's prototype; verify it doesn't modify
9378 // errno.
9379 if (I.onlyReadsMemory()) {
9380 SDValue LHS = getValue(I.getArgOperand(0));
9381 SDValue RHS = getValue(I.getArgOperand(1));
9383 LHS.getValueType(), LHS, RHS));
9384 return;
9385 }
9386 break;
9387 case LibFunc_fabs:
9388 case LibFunc_fabsf:
9389 case LibFunc_fabsl:
9390 if (visitUnaryFloatCall(I, ISD::FABS))
9391 return;
9392 break;
9393 case LibFunc_fmin:
9394 case LibFunc_fminf:
9395 case LibFunc_fminl:
9396 if (visitBinaryFloatCall(I, ISD::FMINNUM))
9397 return;
9398 break;
9399 case LibFunc_fmax:
9400 case LibFunc_fmaxf:
9401 case LibFunc_fmaxl:
9402 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
9403 return;
9404 break;
9405 case LibFunc_fminimum_num:
9406 case LibFunc_fminimum_numf:
9407 case LibFunc_fminimum_numl:
9408 if (visitBinaryFloatCall(I, ISD::FMINIMUMNUM))
9409 return;
9410 break;
9411 case LibFunc_fmaximum_num:
9412 case LibFunc_fmaximum_numf:
9413 case LibFunc_fmaximum_numl:
9414 if (visitBinaryFloatCall(I, ISD::FMAXIMUMNUM))
9415 return;
9416 break;
9417 case LibFunc_sin:
9418 case LibFunc_sinf:
9419 case LibFunc_sinl:
9420 if (visitUnaryFloatCall(I, ISD::FSIN))
9421 return;
9422 break;
9423 case LibFunc_cos:
9424 case LibFunc_cosf:
9425 case LibFunc_cosl:
9426 if (visitUnaryFloatCall(I, ISD::FCOS))
9427 return;
9428 break;
9429 case LibFunc_tan:
9430 case LibFunc_tanf:
9431 case LibFunc_tanl:
9432 if (visitUnaryFloatCall(I, ISD::FTAN))
9433 return;
9434 break;
9435 case LibFunc_asin:
9436 case LibFunc_asinf:
9437 case LibFunc_asinl:
9438 if (visitUnaryFloatCall(I, ISD::FASIN))
9439 return;
9440 break;
9441 case LibFunc_acos:
9442 case LibFunc_acosf:
9443 case LibFunc_acosl:
9444 if (visitUnaryFloatCall(I, ISD::FACOS))
9445 return;
9446 break;
9447 case LibFunc_atan:
9448 case LibFunc_atanf:
9449 case LibFunc_atanl:
9450 if (visitUnaryFloatCall(I, ISD::FATAN))
9451 return;
9452 break;
9453 case LibFunc_atan2:
9454 case LibFunc_atan2f:
9455 case LibFunc_atan2l:
9456 if (visitBinaryFloatCall(I, ISD::FATAN2))
9457 return;
9458 break;
9459 case LibFunc_sinh:
9460 case LibFunc_sinhf:
9461 case LibFunc_sinhl:
9462 if (visitUnaryFloatCall(I, ISD::FSINH))
9463 return;
9464 break;
9465 case LibFunc_cosh:
9466 case LibFunc_coshf:
9467 case LibFunc_coshl:
9468 if (visitUnaryFloatCall(I, ISD::FCOSH))
9469 return;
9470 break;
9471 case LibFunc_tanh:
9472 case LibFunc_tanhf:
9473 case LibFunc_tanhl:
9474 if (visitUnaryFloatCall(I, ISD::FTANH))
9475 return;
9476 break;
9477 case LibFunc_sqrt:
9478 case LibFunc_sqrtf:
9479 case LibFunc_sqrtl:
9480 case LibFunc_sqrt_finite:
9481 case LibFunc_sqrtf_finite:
9482 case LibFunc_sqrtl_finite:
9483 if (visitUnaryFloatCall(I, ISD::FSQRT))
9484 return;
9485 break;
9486 case LibFunc_floor:
9487 case LibFunc_floorf:
9488 case LibFunc_floorl:
9489 if (visitUnaryFloatCall(I, ISD::FFLOOR))
9490 return;
9491 break;
9492 case LibFunc_nearbyint:
9493 case LibFunc_nearbyintf:
9494 case LibFunc_nearbyintl:
9495 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
9496 return;
9497 break;
9498 case LibFunc_ceil:
9499 case LibFunc_ceilf:
9500 case LibFunc_ceill:
9501 if (visitUnaryFloatCall(I, ISD::FCEIL))
9502 return;
9503 break;
9504 case LibFunc_rint:
9505 case LibFunc_rintf:
9506 case LibFunc_rintl:
9507 if (visitUnaryFloatCall(I, ISD::FRINT))
9508 return;
9509 break;
9510 case LibFunc_round:
9511 case LibFunc_roundf:
9512 case LibFunc_roundl:
9513 if (visitUnaryFloatCall(I, ISD::FROUND))
9514 return;
9515 break;
9516 case LibFunc_trunc:
9517 case LibFunc_truncf:
9518 case LibFunc_truncl:
9519 if (visitUnaryFloatCall(I, ISD::FTRUNC))
9520 return;
9521 break;
9522 case LibFunc_log2:
9523 case LibFunc_log2f:
9524 case LibFunc_log2l:
9525 if (visitUnaryFloatCall(I, ISD::FLOG2))
9526 return;
9527 break;
9528 case LibFunc_exp2:
9529 case LibFunc_exp2f:
9530 case LibFunc_exp2l:
9531 if (visitUnaryFloatCall(I, ISD::FEXP2))
9532 return;
9533 break;
9534 case LibFunc_exp10:
9535 case LibFunc_exp10f:
9536 case LibFunc_exp10l:
9537 if (visitUnaryFloatCall(I, ISD::FEXP10))
9538 return;
9539 break;
9540 case LibFunc_ldexp:
9541 case LibFunc_ldexpf:
9542 case LibFunc_ldexpl:
9543 if (visitBinaryFloatCall(I, ISD::FLDEXP))
9544 return;
9545 break;
9546 case LibFunc_memcmp:
9547 if (visitMemCmpBCmpCall(I))
9548 return;
9549 break;
9550 case LibFunc_mempcpy:
9551 if (visitMemPCpyCall(I))
9552 return;
9553 break;
9554 case LibFunc_memchr:
9555 if (visitMemChrCall(I))
9556 return;
9557 break;
9558 case LibFunc_strcpy:
9559 if (visitStrCpyCall(I, false))
9560 return;
9561 break;
9562 case LibFunc_stpcpy:
9563 if (visitStrCpyCall(I, true))
9564 return;
9565 break;
9566 case LibFunc_strcmp:
9567 if (visitStrCmpCall(I))
9568 return;
9569 break;
9570 case LibFunc_strlen:
9571 if (visitStrLenCall(I))
9572 return;
9573 break;
9574 case LibFunc_strnlen:
9575 if (visitStrNLenCall(I))
9576 return;
9577 break;
9578 }
9579 }
9580 }
9581
9582 if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
9583 LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
9584 return;
9585 }
9586
9587 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
9588 // have to do anything here to lower funclet bundles.
9589 // CFGuardTarget bundles are lowered in LowerCallTo.
9590 assert(!I.hasOperandBundlesOtherThan(
9591 {LLVMContext::OB_deopt, LLVMContext::OB_funclet,
9592 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated,
9593 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi,
9594 LLVMContext::OB_convergencectrl}) &&
9595 "Cannot lower calls with arbitrary operand bundles!");
9596
9597 SDValue Callee = getValue(I.getCalledOperand());
9598
9599 if (I.hasDeoptState())
9600 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
9601 else
9602 // Check if we can potentially perform a tail call. More detailed checking
9603 // is be done within LowerCallTo, after more information about the call is
9604 // known.
9605 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
9606}
9607
9609 const CallBase &CB, const BasicBlock *EHPadBB) {
9610 auto PAB = CB.getOperandBundle("ptrauth");
9611 const Value *CalleeV = CB.getCalledOperand();
9612
9613 // Gather the call ptrauth data from the operand bundle:
9614 // [ i32 <key>, i64 <discriminator> ]
9615 const auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
9616 const Value *Discriminator = PAB->Inputs[1];
9617
9618 assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
9619 assert(Discriminator->getType()->isIntegerTy(64) &&
9620 "Invalid ptrauth discriminator");
9621
9622 // Look through ptrauth constants to find the raw callee.
9623 // Do a direct unauthenticated call if we found it and everything matches.
9624 if (const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CalleeV))
9625 if (CalleeCPA->isKnownCompatibleWith(Key, Discriminator,
9626 DAG.getDataLayout()))
9627 return LowerCallTo(CB, getValue(CalleeCPA->getPointer()), CB.isTailCall(),
9628 CB.isMustTailCall(), EHPadBB);
9629
9630 // Functions should never be ptrauth-called directly.
9631 assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
9632
9633 // Otherwise, do an authenticated indirect call.
9634 TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
9635 getValue(Discriminator)};
9636
9637 LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
9638 EHPadBB, &PAI);
9639}
9640
9641namespace {
9642
9643/// AsmOperandInfo - This contains information for each constraint that we are
9644/// lowering.
9645class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
9646public:
9647 /// CallOperand - If this is the result output operand or a clobber
9648 /// this is null, otherwise it is the incoming operand to the CallInst.
9649 /// This gets modified as the asm is processed.
9650 SDValue CallOperand;
9651
9652 /// AssignedRegs - If this is a register or register class operand, this
9653 /// contains the set of register corresponding to the operand.
9654 RegsForValue AssignedRegs;
9655
9656 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
9657 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
9658 }
9659
9660 /// Whether or not this operand accesses memory
9661 bool hasMemory(const TargetLowering &TLI) const {
9662 // Indirect operand accesses access memory.
9663 if (isIndirect)
9664 return true;
9665
9666 for (const auto &Code : Codes)
9668 return true;
9669
9670 return false;
9671 }
9672};
9673
9674
9675} // end anonymous namespace
9676
9677/// Make sure that the output operand \p OpInfo and its corresponding input
9678/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
9679/// out).
9680static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
9681 SDISelAsmOperandInfo &MatchingOpInfo,
9682 SelectionDAG &DAG) {
9683 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
9684 return;
9685
9687 const auto &TLI = DAG.getTargetLoweringInfo();
9688
9689 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
9690 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
9691 OpInfo.ConstraintVT);
9692 std::pair<unsigned, const TargetRegisterClass *> InputRC =
9693 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
9694 MatchingOpInfo.ConstraintVT);
9695 const bool OutOpIsIntOrFP =
9696 OpInfo.ConstraintVT.isInteger() || OpInfo.ConstraintVT.isFloatingPoint();
9697 const bool InOpIsIntOrFP = MatchingOpInfo.ConstraintVT.isInteger() ||
9698 MatchingOpInfo.ConstraintVT.isFloatingPoint();
9699 if ((OutOpIsIntOrFP != InOpIsIntOrFP) || (MatchRC.second != InputRC.second)) {
9700 // FIXME: error out in a more elegant fashion
9701 report_fatal_error("Unsupported asm: input constraint"
9702 " with a matching output constraint of"
9703 " incompatible type!");
9704 }
9705 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
9706}
9707
9708/// Get a direct memory input to behave well as an indirect operand.
9709/// This may introduce stores, hence the need for a \p Chain.
9710/// \return The (possibly updated) chain.
9711static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
9712 SDISelAsmOperandInfo &OpInfo,
9713 SelectionDAG &DAG) {
9714 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9715
9716 // If we don't have an indirect input, put it in the constpool if we can,
9717 // otherwise spill it to a stack slot.
9718 // TODO: This isn't quite right. We need to handle these according to
9719 // the addressing mode that the constraint wants. Also, this may take
9720 // an additional register for the computation and we don't want that
9721 // either.
9722
9723 // If the operand is a float, integer, or vector constant, spill to a
9724 // constant pool entry to get its address.
9725 const Value *OpVal = OpInfo.CallOperandVal;
9726 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
9727 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
9728 OpInfo.CallOperand = DAG.getConstantPool(
9729 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
9730 return Chain;
9731 }
9732
9733 // Otherwise, create a stack slot and emit a store to it before the asm.
9734 Type *Ty = OpVal->getType();
9735 auto &DL = DAG.getDataLayout();
9736 TypeSize TySize = DL.getTypeAllocSize(Ty);
9739 int StackID = 0;
9740 if (TySize.isScalable())
9741 StackID = TFI->getStackIDForScalableVectors();
9742 int SSFI = MF.getFrameInfo().CreateStackObject(TySize.getKnownMinValue(),
9743 DL.getPrefTypeAlign(Ty), false,
9744 nullptr, StackID);
9745 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
9746 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot,
9748 TLI.getMemValueType(DL, Ty));
9749 OpInfo.CallOperand = StackSlot;
9750
9751 return Chain;
9752}
9753
9754/// GetRegistersForValue - Assign registers (virtual or physical) for the
9755/// specified operand. We prefer to assign virtual registers, to allow the
9756/// register allocator to handle the assignment process. However, if the asm
9757/// uses features that we can't model on machineinstrs, we have SDISel do the
9758/// allocation. This produces generally horrible, but correct, code.
9759///
9760/// OpInfo describes the operand
9761/// RefOpInfo describes the matching operand if any, the operand otherwise
9762static std::optional<unsigned>
9764 SDISelAsmOperandInfo &OpInfo,
9765 SDISelAsmOperandInfo &RefOpInfo) {
9766 LLVMContext &Context = *DAG.getContext();
9767 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9768
9772
9773 // No work to do for memory/address operands.
9774 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
9775 OpInfo.ConstraintType == TargetLowering::C_Address)
9776 return std::nullopt;
9777
9778 // If this is a constraint for a single physreg, or a constraint for a
9779 // register class, find it.
9780 unsigned AssignedReg;
9781 const TargetRegisterClass *RC;
9782 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
9783 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
9784 // RC is unset only on failure. Return immediately.
9785 if (!RC)
9786 return std::nullopt;
9787
9788 // Get the actual register value type. This is important, because the user
9789 // may have asked for (e.g.) the AX register in i32 type. We need to
9790 // remember that AX is actually i16 to get the right extension.
9791 const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
9792
9793 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) {
9794 // If this is an FP operand in an integer register (or visa versa), or more
9795 // generally if the operand value disagrees with the register class we plan
9796 // to stick it in, fix the operand type.
9797 //
9798 // If this is an input value, the bitcast to the new type is done now.
9799 // Bitcast for output value is done at the end of visitInlineAsm().
9800 if ((OpInfo.Type == InlineAsm::isOutput ||
9801 OpInfo.Type == InlineAsm::isInput) &&
9802 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
9803 // Try to convert to the first EVT that the reg class contains. If the
9804 // types are identical size, use a bitcast to convert (e.g. two differing
9805 // vector types). Note: output bitcast is done at the end of
9806 // visitInlineAsm().
9807 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
9808 // Exclude indirect inputs while they are unsupported because the code
9809 // to perform the load is missing and thus OpInfo.CallOperand still
9810 // refers to the input address rather than the pointed-to value.
9811 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
9812 OpInfo.CallOperand =
9813 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
9814 OpInfo.ConstraintVT = RegVT;
9815 // If the operand is an FP value and we want it in integer registers,
9816 // use the corresponding integer type. This turns an f64 value into
9817 // i64, which can be passed with two i32 values on a 32-bit machine.
9818 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
9819 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
9820 if (OpInfo.Type == InlineAsm::isInput)
9821 OpInfo.CallOperand =
9822 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
9823 OpInfo.ConstraintVT = VT;
9824 }
9825 }
9826 }
9827
9828 // No need to allocate a matching input constraint since the constraint it's
9829 // matching to has already been allocated.
9830 if (OpInfo.isMatchingInputConstraint())
9831 return std::nullopt;
9832
9833 EVT ValueVT = OpInfo.ConstraintVT;
9834 if (OpInfo.ConstraintVT == MVT::Other)
9835 ValueVT = RegVT;
9836
9837 // Initialize NumRegs.
9838 unsigned NumRegs = 1;
9839 if (OpInfo.ConstraintVT != MVT::Other)
9840 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT);
9841
9842 // If this is a constraint for a specific physical register, like {r17},
9843 // assign it now.
9844
9845 // If this associated to a specific register, initialize iterator to correct
9846 // place. If virtual, make sure we have enough registers
9847
9848 // Initialize iterator if necessary
9851
9852 // Do not check for single registers.
9853 if (AssignedReg) {
9854 I = std::find(I, RC->end(), AssignedReg);
9855 if (I == RC->end()) {
9856 // RC does not contain the selected register, which indicates a
9857 // mismatch between the register and the required type/bitwidth.
9858 return {AssignedReg};
9859 }
9860 }
9861
9862 for (; NumRegs; --NumRegs, ++I) {
9863 assert(I != RC->end() && "Ran out of registers to allocate!");
9864 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC);
9865 Regs.push_back(R);
9866 }
9867
9868 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
9869 return std::nullopt;
9870}
9871
9872static unsigned
9874 const std::vector<SDValue> &AsmNodeOperands) {
9875 // Scan until we find the definition we already emitted of this operand.
9876 unsigned CurOp = InlineAsm::Op_FirstOperand;
9877 for (; OperandNo; --OperandNo) {
9878 // Advance to the next operand.
9879 unsigned OpFlag = AsmNodeOperands[CurOp]->getAsZExtVal();
9880 const InlineAsm::Flag F(OpFlag);
9881 assert(
9882 (F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isMemKind()) &&
9883 "Skipped past definitions?");
9884 CurOp += F.getNumOperandRegisters() + 1;
9885 }
9886 return CurOp;
9887}
9888
9889namespace {
9890
9891class ExtraFlags {
9892 unsigned Flags = 0;
9893
9894public:
9895 explicit ExtraFlags(const CallBase &Call) {
9896 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9897 if (IA->hasSideEffects())
9899 if (IA->isAlignStack())
9901 if (Call.isConvergent())
9903 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
9904 }
9905
9906 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
9907 // Ideally, we would only check against memory constraints. However, the
9908 // meaning of an Other constraint can be target-specific and we can't easily
9909 // reason about it. Therefore, be conservative and set MayLoad/MayStore
9910 // for Other constraints as well.
9913 if (OpInfo.Type == InlineAsm::isInput)
9915 else if (OpInfo.Type == InlineAsm::isOutput)
9917 else if (OpInfo.Type == InlineAsm::isClobber)
9919 }
9920 }
9921
9922 unsigned get() const { return Flags; }
9923};
9924
9925} // end anonymous namespace
9926
9927static bool isFunction(SDValue Op) {
9928 if (Op && Op.getOpcode() == ISD::GlobalAddress) {
9929 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
9930 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal());
9931
9932 // In normal "call dllimport func" instruction (non-inlineasm) it force
9933 // indirect access by specifing call opcode. And usually specially print
9934 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can
9935 // not do in this way now. (In fact, this is similar with "Data Access"
9936 // action). So here we ignore dllimport function.
9937 if (Fn && !Fn->hasDLLImportStorageClass())
9938 return true;
9939 }
9940 }
9941 return false;
9942}
9943
9944/// visitInlineAsm - Handle a call to an InlineAsm object.
9945void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call,
9946 const BasicBlock *EHPadBB) {
9947 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
9948
9949 /// ConstraintOperands - Information about all of the constraints.
9950 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands;
9951
9955
9956 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
9957 // AsmDialect, MayLoad, MayStore).
9958 bool HasSideEffect = IA->hasSideEffects();
9959 ExtraFlags ExtraInfo(Call);
9960
9961 for (auto &T : TargetConstraints) {
9962 ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
9963 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
9964
9965 if (OpInfo.CallOperandVal)
9966 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
9967
9968 if (!HasSideEffect)
9969 HasSideEffect = OpInfo.hasMemory(TLI);
9970
9971 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
9972 // FIXME: Could we compute this on OpInfo rather than T?
9973
9974 // Compute the constraint code and ConstraintType to use.
9976
9977 if (T.ConstraintType == TargetLowering::C_Immediate &&
9978 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand))
9979 // We've delayed emitting a diagnostic like the "n" constraint because
9980 // inlining could cause an integer showing up.
9981 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) +
9982 "' expects an integer constant "
9983 "expression");
9984
9985 ExtraInfo.update(T);
9986 }
9987
9988 // We won't need to flush pending loads if this asm doesn't touch
9989 // memory and is nonvolatile.
9990 SDValue Glue, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
9991
9992 bool EmitEHLabels = isa<InvokeInst>(Call);
9993 if (EmitEHLabels) {
9994 assert(EHPadBB && "InvokeInst must have an EHPadBB");
9995 }
9996 bool IsCallBr = isa<CallBrInst>(Call);
9997
9998 if (IsCallBr || EmitEHLabels) {
9999 // If this is a callbr or invoke we need to flush pending exports since
10000 // inlineasm_br and invoke are terminators.
10001 // We need to do this before nodes are glued to the inlineasm_br node.
10002 Chain = getControlRoot();
10003 }
10004
10005 MCSymbol *BeginLabel = nullptr;
10006 if (EmitEHLabels) {
10007 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel);
10008 }
10009
10010 int OpNo = -1;
10011 SmallVector<StringRef> AsmStrs;
10012 IA->collectAsmStrs(AsmStrs);
10013
10014 // Second pass over the constraints: compute which constraint option to use.
10015 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10016 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput)
10017 OpNo++;
10018
10019 // If this is an output operand with a matching input operand, look up the
10020 // matching input. If their types mismatch, e.g. one is an integer, the
10021 // other is floating point, or their sizes are different, flag it as an
10022 // error.
10023 if (OpInfo.hasMatchingInput()) {
10024 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
10025 patchMatchingInput(OpInfo, Input, DAG);
10026 }
10027
10028 // Compute the constraint code and ConstraintType to use.
10029 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
10030
10031 if ((OpInfo.ConstraintType == TargetLowering::C_Memory &&
10032 OpInfo.Type == InlineAsm::isClobber) ||
10033 OpInfo.ConstraintType == TargetLowering::C_Address)
10034 continue;
10035
10036 // In Linux PIC model, there are 4 cases about value/label addressing:
10037 //
10038 // 1: Function call or Label jmp inside the module.
10039 // 2: Data access (such as global variable, static variable) inside module.
10040 // 3: Function call or Label jmp outside the module.
10041 // 4: Data access (such as global variable) outside the module.
10042 //
10043 // Due to current llvm inline asm architecture designed to not "recognize"
10044 // the asm code, there are quite troubles for us to treat mem addressing
10045 // differently for same value/adress used in different instuctions.
10046 // For example, in pic model, call a func may in plt way or direclty
10047 // pc-related, but lea/mov a function adress may use got.
10048 //
10049 // Here we try to "recognize" function call for the case 1 and case 3 in
10050 // inline asm. And try to adjust the constraint for them.
10051 //
10052 // TODO: Due to current inline asm didn't encourage to jmp to the outsider
10053 // label, so here we don't handle jmp function label now, but we need to
10054 // enhance it (especilly in PIC model) if we meet meaningful requirements.
10055 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) &&
10056 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) &&
10058 OpInfo.isIndirect = false;
10059 OpInfo.ConstraintType = TargetLowering::C_Address;
10060 }
10061
10062 // If this is a memory input, and if the operand is not indirect, do what we
10063 // need to provide an address for the memory input.
10064 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
10065 !OpInfo.isIndirect) {
10066 assert((OpInfo.isMultipleAlternative ||
10067 (OpInfo.Type == InlineAsm::isInput)) &&
10068 "Can only indirectify direct input operands!");
10069
10070 // Memory operands really want the address of the value.
10071 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
10072
10073 // There is no longer a Value* corresponding to this operand.
10074 OpInfo.CallOperandVal = nullptr;
10075
10076 // It is now an indirect operand.
10077 OpInfo.isIndirect = true;
10078 }
10079
10080 }
10081
10082 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
10083 std::vector<SDValue> AsmNodeOperands;
10084 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
10085 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
10086 IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout())));
10087
10088 // If we have a !srcloc metadata node associated with it, we want to attach
10089 // this to the ultimately generated inline asm machineinstr. To do this, we
10090 // pass in the third operand as this (potentially null) inline asm MDNode.
10091 const MDNode *SrcLoc = Call.getMetadata("srcloc");
10092 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
10093
10094 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
10095 // bits as operand 3.
10096 AsmNodeOperands.push_back(DAG.getTargetConstant(
10097 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
10098
10099 // Third pass: Loop over operands to prepare DAG-level operands.. As part of
10100 // this, assign virtual and physical registers for inputs and otput.
10101 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10102 // Assign Registers.
10103 SDISelAsmOperandInfo &RefOpInfo =
10104 OpInfo.isMatchingInputConstraint()
10105 ? ConstraintOperands[OpInfo.getMatchedOperand()]
10106 : OpInfo;
10107 const auto RegError =
10108 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
10109 if (RegError) {
10112 const char *RegName = TRI.getName(*RegError);
10113 emitInlineAsmError(Call, "register '" + Twine(RegName) +
10114 "' allocated for constraint '" +
10115 Twine(OpInfo.ConstraintCode) +
10116 "' does not match required type");
10117 return;
10118 }
10119
10120 auto DetectWriteToReservedRegister = [&]() {
10123 for (unsigned Reg : OpInfo.AssignedRegs.Regs) {
10125 TRI.isInlineAsmReadOnlyReg(MF, Reg)) {
10126 const char *RegName = TRI.getName(Reg);
10127 emitInlineAsmError(Call, "write to reserved register '" +
10128 Twine(RegName) + "'");
10129 return true;
10130 }
10131 }
10132 return false;
10133 };
10134 assert((OpInfo.ConstraintType != TargetLowering::C_Address ||
10135 (OpInfo.Type == InlineAsm::isInput &&
10136 !OpInfo.isMatchingInputConstraint())) &&
10137 "Only address as input operand is allowed.");
10138
10139 switch (OpInfo.Type) {
10141 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10142 const InlineAsm::ConstraintCode ConstraintID =
10143 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10145 "Failed to convert memory constraint code to constraint id.");
10146
10147 // Add information to the INLINEASM node to know about this output.
10149 OpFlags.setMemConstraint(ConstraintID);
10150 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
10151 MVT::i32));
10152 AsmNodeOperands.push_back(OpInfo.CallOperand);
10153 } else {
10154 // Otherwise, this outputs to a register (directly for C_Register /
10155 // C_RegisterClass, and a target-defined fashion for
10156 // C_Immediate/C_Other). Find a register that we can use.
10157 if (OpInfo.AssignedRegs.Regs.empty()) {
10158 emitInlineAsmError(
10159 Call, "couldn't allocate output register for constraint '" +
10160 Twine(OpInfo.ConstraintCode) + "'");
10161 return;
10162 }
10163
10164 if (DetectWriteToReservedRegister())
10165 return;
10166
10167 // Add information to the INLINEASM node to know that this register is
10168 // set.
10169 OpInfo.AssignedRegs.AddInlineAsmOperands(
10170 OpInfo.isEarlyClobber ? InlineAsm::Kind::RegDefEarlyClobber
10172 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
10173 }
10174 break;
10175
10176 case InlineAsm::isInput:
10177 case InlineAsm::isLabel: {
10178 SDValue InOperandVal = OpInfo.CallOperand;
10179
10180 if (OpInfo.isMatchingInputConstraint()) {
10181 // If this is required to match an output register we have already set,
10182 // just use its register.
10183 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
10184 AsmNodeOperands);
10185 InlineAsm::Flag Flag(AsmNodeOperands[CurOp]->getAsZExtVal());
10186 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
10187 if (OpInfo.isIndirect) {
10188 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
10189 emitInlineAsmError(Call, "inline asm not supported yet: "
10190 "don't know how to handle tied "
10191 "indirect register inputs");
10192 return;
10193 }
10194
10199 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]);
10200 Register TiedReg = R->getReg();
10201 MVT RegVT = R->getSimpleValueType(0);
10202 const TargetRegisterClass *RC =
10203 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg)
10204 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT)
10205 : TRI.getMinimalPhysRegClass(TiedReg);
10206 for (unsigned i = 0, e = Flag.getNumOperandRegisters(); i != e; ++i)
10207 Regs.push_back(MRI.createVirtualRegister(RC));
10208
10209 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
10210
10211 SDLoc dl = getCurSDLoc();
10212 // Use the produced MatchedRegs object to
10213 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue, &Call);
10214 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, true,
10215 OpInfo.getMatchedOperand(), dl, DAG,
10216 AsmNodeOperands);
10217 break;
10218 }
10219
10220 assert(Flag.isMemKind() && "Unknown matching constraint!");
10221 assert(Flag.getNumOperandRegisters() == 1 &&
10222 "Unexpected number of operands");
10223 // Add information to the INLINEASM node to know about this input.
10224 // See InlineAsm.h isUseOperandTiedToDef.
10225 Flag.clearMemConstraint();
10226 Flag.setMatchingOp(OpInfo.getMatchedOperand());
10227 AsmNodeOperands.push_back(DAG.getTargetConstant(
10228 Flag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
10229 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
10230 break;
10231 }
10232
10233 // Treat indirect 'X' constraint as memory.
10234 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
10235 OpInfo.isIndirect)
10236 OpInfo.ConstraintType = TargetLowering::C_Memory;
10237
10238 if (OpInfo.ConstraintType == TargetLowering::C_Immediate ||
10239 OpInfo.ConstraintType == TargetLowering::C_Other) {
10240 std::vector<SDValue> Ops;
10241 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
10242 Ops, DAG);
10243 if (Ops.empty()) {
10244 if (OpInfo.ConstraintType == TargetLowering::C_Immediate)
10245 if (isa<ConstantSDNode>(InOperandVal)) {
10246 emitInlineAsmError(Call, "value out of range for constraint '" +
10247 Twine(OpInfo.ConstraintCode) + "'");
10248 return;
10249 }
10250
10251 emitInlineAsmError(Call,
10252 "invalid operand for inline asm constraint '" +
10253 Twine(OpInfo.ConstraintCode) + "'");
10254 return;
10255 }
10256
10257 // Add information to the INLINEASM node to know about this input.
10258 InlineAsm::Flag ResOpType(InlineAsm::Kind::Imm, Ops.size());
10259 AsmNodeOperands.push_back(DAG.getTargetConstant(
10260 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
10261 llvm::append_range(AsmNodeOperands, Ops);
10262 break;
10263 }
10264
10265 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
10266 assert((OpInfo.isIndirect ||
10267 OpInfo.ConstraintType != TargetLowering::C_Memory) &&
10268 "Operand must be indirect to be a mem!");
10269 assert(InOperandVal.getValueType() ==
10271 "Memory operands expect pointer values");
10272
10273 const InlineAsm::ConstraintCode ConstraintID =
10274 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10276 "Failed to convert memory constraint code to constraint id.");
10277
10278 // Add information to the INLINEASM node to know about this input.
10280 ResOpType.setMemConstraint(ConstraintID);
10281 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
10282 getCurSDLoc(),
10283 MVT::i32));
10284 AsmNodeOperands.push_back(InOperandVal);
10285 break;
10286 }
10287
10288 if (OpInfo.ConstraintType == TargetLowering::C_Address) {
10289 const InlineAsm::ConstraintCode ConstraintID =
10290 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
10292 "Failed to convert memory constraint code to constraint id.");
10293
10295
10296 SDValue AsmOp = InOperandVal;
10297 if (isFunction(InOperandVal)) {
10298 auto *GA = cast<GlobalAddressSDNode>(InOperandVal);
10299 ResOpType = InlineAsm::Flag(InlineAsm::Kind::Func, 1);
10300 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(),
10301 InOperandVal.getValueType(),
10302 GA->getOffset());
10303 }
10304
10305 // Add information to the INLINEASM node to know about this input.
10306 ResOpType.setMemConstraint(ConstraintID);
10307
10308 AsmNodeOperands.push_back(
10309 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32));
10310
10311 AsmNodeOperands.push_back(AsmOp);
10312 break;
10313 }
10314
10315 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
10316 OpInfo.ConstraintType != TargetLowering::C_Register) {
10317 emitInlineAsmError(Call, "unknown asm constraint '" +
10318 Twine(OpInfo.ConstraintCode) + "'");
10319 return;
10320 }
10321
10322 // TODO: Support this.
10323 if (OpInfo.isIndirect) {
10324 emitInlineAsmError(
10325 Call, "Don't know how to handle indirect register inputs yet "
10326 "for constraint '" +
10327 Twine(OpInfo.ConstraintCode) + "'");
10328 return;
10329 }
10330
10331 // Copy the input into the appropriate registers.
10332 if (OpInfo.AssignedRegs.Regs.empty()) {
10333 emitInlineAsmError(Call,
10334 "couldn't allocate input reg for constraint '" +
10335 Twine(OpInfo.ConstraintCode) + "'");
10336 return;
10337 }
10338
10339 if (DetectWriteToReservedRegister())
10340 return;
10341
10342 SDLoc dl = getCurSDLoc();
10343
10344 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Glue,
10345 &Call);
10346
10347 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::RegUse, false,
10348 0, dl, DAG, AsmNodeOperands);
10349 break;
10350 }
10352 // Add the clobbered value to the operand list, so that the register
10353 // allocator is aware that the physreg got clobbered.
10354 if (!OpInfo.AssignedRegs.Regs.empty())
10355 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind::Clobber,
10356 false, 0, getCurSDLoc(), DAG,
10357 AsmNodeOperands);
10358 break;
10359 }
10360 }
10361
10362 // Finish up input operands. Set the input chain and add the flag last.
10363 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
10364 if (Glue.getNode()) AsmNodeOperands.push_back(Glue);
10365
10366 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM;
10367 Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
10368 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
10369 Glue = Chain.getValue(1);
10370
10371 // Do additional work to generate outputs.
10372
10373 SmallVector<EVT, 1> ResultVTs;
10374 SmallVector<SDValue, 1> ResultValues;
10375 SmallVector<SDValue, 8> OutChains;
10376
10377 llvm::Type *CallResultType = Call.getType();
10378 ArrayRef<Type *> ResultTypes;
10379 if (StructType *StructResult = dyn_cast<StructType>(CallResultType))
10380 ResultTypes = StructResult->elements();
10381 else if (!CallResultType->isVoidTy())
10382 ResultTypes = ArrayRef(CallResultType);
10383
10384 auto CurResultType = ResultTypes.begin();
10385 auto handleRegAssign = [&](SDValue V) {
10386 assert(CurResultType != ResultTypes.end() && "Unexpected value");
10387 assert((*CurResultType)->isSized() && "Unexpected unsized type");
10388 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
10389 ++CurResultType;
10390 // If the type of the inline asm call site return value is different but has
10391 // same size as the type of the asm output bitcast it. One example of this
10392 // is for vectors with different width / number of elements. This can
10393 // happen for register classes that can contain multiple different value
10394 // types. The preg or vreg allocated may not have the same VT as was
10395 // expected.
10396 //
10397 // This can also happen for a return value that disagrees with the register
10398 // class it is put in, eg. a double in a general-purpose register on a
10399 // 32-bit machine.
10400 if (ResultVT != V.getValueType() &&
10401 ResultVT.getSizeInBits() == V.getValueSizeInBits())
10402 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
10403 else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
10404 V.getValueType().isInteger()) {
10405 // If a result value was tied to an input value, the computed result
10406 // may have a wider width than the expected result. Extract the
10407 // relevant portion.
10408 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
10409 }
10410 assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
10411 ResultVTs.push_back(ResultVT);
10412 ResultValues.push_back(V);
10413 };
10414
10415 // Deal with output operands.
10416 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
10417 if (OpInfo.Type == InlineAsm::isOutput) {
10418 SDValue Val;
10419 // Skip trivial output operands.
10420 if (OpInfo.AssignedRegs.Regs.empty())
10421 continue;
10422
10423 switch (OpInfo.ConstraintType) {
10426 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
10427 Chain, &Glue, &Call);
10428 break;
10431 Val = TLI.LowerAsmOutputForConstraint(Chain, Glue, getCurSDLoc(),
10432 OpInfo, DAG);
10433 break;
10435 break; // Already handled.
10437 break; // Silence warning.
10439 assert(false && "Unexpected unknown constraint");
10440 }
10441
10442 // Indirect output manifest as stores. Record output chains.
10443 if (OpInfo.isIndirect) {
10444 const Value *Ptr = OpInfo.CallOperandVal;
10445 assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
10446 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
10448 OutChains.push_back(Store);
10449 } else {
10450 // generate CopyFromRegs to associated registers.
10451 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
10452 if (Val.getOpcode() == ISD::MERGE_VALUES) {
10453 for (const SDValue &V : Val->op_values())
10454 handleRegAssign(V);
10455 } else
10456 handleRegAssign(Val);
10457 }
10458 }
10459 }
10460
10461 // Set results.
10462 if (!ResultValues.empty()) {
10463 assert(CurResultType == ResultTypes.end() &&
10464 "Mismatch in number of ResultTypes");
10465 assert(ResultValues.size() == ResultTypes.size() &&
10466 "Mismatch in number of output operands in asm result");
10467
10469 DAG.getVTList(ResultVTs), ResultValues);
10470 setValue(&Call, V);
10471 }
10472
10473 // Collect store chains.
10474 if (!OutChains.empty())
10475 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
10476
10477 if (EmitEHLabels) {
10478 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel);
10479 }
10480
10481 // Only Update Root if inline assembly has a memory effect.
10482 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr ||
10483 EmitEHLabels)
10484 DAG.setRoot(Chain);
10485}
10486
10487void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call,
10488 const Twine &Message) {
10489 LLVMContext &Ctx = *DAG.getContext();
10490 Ctx.diagnose(DiagnosticInfoInlineAsm(Call, Message));
10491
10492 // Make sure we leave the DAG in a valid state
10494 SmallVector<EVT, 1> ValueVTs;
10495 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs);
10496
10497 if (ValueVTs.empty())
10498 return;
10499
10501 for (const EVT &VT : ValueVTs)
10502 Ops.push_back(DAG.getUNDEF(VT));
10503
10504 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc()));
10505}
10506
10507void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
10509 MVT::Other, getRoot(),
10510 getValue(I.getArgOperand(0)),
10511 DAG.getSrcValue(I.getArgOperand(0))));
10512}
10513
10514void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
10516 const DataLayout &DL = DAG.getDataLayout();
10518 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(),
10519 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)),
10520 DL.getABITypeAlign(I.getType()).value());
10521 DAG.setRoot(V.getValue(1));
10522
10523 if (I.getType()->isPointerTy())
10525 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType()));
10526 setValue(&I, V);
10527}
10528
10529void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
10531 MVT::Other, getRoot(),
10532 getValue(I.getArgOperand(0)),
10533 DAG.getSrcValue(I.getArgOperand(0))));
10534}
10535
10536void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
10538 MVT::Other, getRoot(),
10539 getValue(I.getArgOperand(0)),
10540 getValue(I.getArgOperand(1)),
10541 DAG.getSrcValue(I.getArgOperand(0)),
10542 DAG.getSrcValue(I.getArgOperand(1))));
10543}
10544
10546 const Instruction &I,
10547 SDValue Op) {
10548 std::optional<ConstantRange> CR = getRange(I);
10549
10550 if (!CR || CR->isFullSet() || CR->isEmptySet() || CR->isUpperWrapped())
10551 return Op;
10552
10553 APInt Lo = CR->getUnsignedMin();
10554 if (!Lo.isMinValue())
10555 return Op;
10556
10557 APInt Hi = CR->getUnsignedMax();
10558 unsigned Bits = std::max(Hi.getActiveBits(),
10559 static_cast<unsigned>(IntegerType::MIN_INT_BITS));
10560
10561 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
10562
10563 SDLoc SL = getCurSDLoc();
10564
10565 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
10566 DAG.getValueType(SmallVT));
10567 unsigned NumVals = Op.getNode()->getNumValues();
10568 if (NumVals == 1)
10569 return ZExt;
10570
10572
10573 Ops.push_back(ZExt);
10574 for (unsigned I = 1; I != NumVals; ++I)
10575 Ops.push_back(Op.getValue(I));
10576
10577 return DAG.getMergeValues(Ops, SL);
10578}
10579
10580/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
10581/// the call being lowered.
10582///
10583/// This is a helper for lowering intrinsics that follow a target calling
10584/// convention or require stack pointer adjustment. Only a subset of the
10585/// intrinsic's operands need to participate in the calling convention.
10588 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
10589 AttributeSet RetAttrs, bool IsPatchPoint) {
10591 Args.reserve(NumArgs);
10592
10593 // Populate the argument list.
10594 // Attributes for args start at offset 1, after the return attribute.
10595 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
10596 ArgI != ArgE; ++ArgI) {
10597 const Value *V = Call->getOperand(ArgI);
10598
10599 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
10600
10602 Entry.Node = getValue(V);
10603 Entry.Ty = V->getType();
10604 Entry.setAttributes(Call, ArgI);
10605 Args.push_back(Entry);
10606 }
10607
10609 .setChain(getRoot())
10610 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args),
10611 RetAttrs)
10612 .setDiscardResult(Call->use_empty())
10613 .setIsPatchPoint(IsPatchPoint)
10615 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0);
10616}
10617
10618/// Add a stack map intrinsic call's live variable operands to a stackmap
10619/// or patchpoint target node's operand list.
10620///
10621/// Constants are converted to TargetConstants purely as an optimization to
10622/// avoid constant materialization and register allocation.
10623///
10624/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
10625/// generate addess computation nodes, and so FinalizeISel can convert the
10626/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
10627/// address materialization and register allocation, but may also be required
10628/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
10629/// alloca in the entry block, then the runtime may assume that the alloca's
10630/// StackMap location can be read immediately after compilation and that the
10631/// location is valid at any point during execution (this is similar to the
10632/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
10633/// only available in a register, then the runtime would need to trap when
10634/// execution reaches the StackMap in order to read the alloca's location.
10635static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx,
10636 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
10637 SelectionDAGBuilder &Builder) {
10638 SelectionDAG &DAG = Builder.DAG;
10639 for (unsigned I = StartIdx; I < Call.arg_size(); I++) {
10640 SDValue Op = Builder.getValue(Call.getArgOperand(I));
10641
10642 // Things on the stack are pointer-typed, meaning that they are already
10643 // legal and can be emitted directly to target nodes.
10644 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
10645 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType()));
10646 } else {
10647 // Otherwise emit a target independent node to be legalised.
10648 Ops.push_back(Builder.getValue(Call.getArgOperand(I)));
10649 }
10650 }
10651}
10652
10653/// Lower llvm.experimental.stackmap.
10654void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
10655 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
10656 // [live variables...])
10657
10658 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.");
10659
10660 SDValue Chain, InGlue, Callee;
10662
10663 SDLoc DL = getCurSDLoc();
10665
10666 // The stackmap intrinsic only records the live variables (the arguments
10667 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
10668 // intrinsic, this won't be lowered to a function call. This means we don't
10669 // have to worry about calling conventions and target specific lowering code.
10670 // Instead we perform the call lowering right here.
10671 //
10672 // chain, flag = CALLSEQ_START(chain, 0, 0)
10673 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
10674 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
10675 //
10676 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
10677 InGlue = Chain.getValue(1);
10678
10679 // Add the STACKMAP operands, starting with DAG house-keeping.
10680 Ops.push_back(Chain);
10681 Ops.push_back(InGlue);
10682
10683 // Add the <id>, <numShadowBytes> operands.
10684 //
10685 // These do not require legalisation, and can be emitted directly to target
10686 // constant nodes.
10688 assert(ID.getValueType() == MVT::i64);
10689 SDValue IDConst =
10690 DAG.getTargetConstant(ID->getAsZExtVal(), DL, ID.getValueType());
10691 Ops.push_back(IDConst);
10692
10693 SDValue Shad = getValue(CI.getArgOperand(1));
10694 assert(Shad.getValueType() == MVT::i32);
10695 SDValue ShadConst =
10697 Ops.push_back(ShadConst);
10698
10699 // Add the live variables.
10700 addStackMapLiveVars(CI, 2, DL, Ops, *this);
10701
10702 // Create the STACKMAP node.
10703 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10704 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops);
10705 InGlue = Chain.getValue(1);
10706
10707 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InGlue, DL);
10708
10709 // Stackmaps don't generate values, so nothing goes into the NodeMap.
10710
10711 // Set the root to the target-lowered call chain.
10712 DAG.setRoot(Chain);
10713
10714 // Inform the Frame Information that we have a stackmap in this function.
10716}
10717
10718/// Lower llvm.experimental.patchpoint directly to its target opcode.
10719void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB,
10720 const BasicBlock *EHPadBB) {
10721 // <ty> @llvm.experimental.patchpoint.<ty>(i64 <id>,
10722 // i32 <numBytes>,
10723 // i8* <target>,
10724 // i32 <numArgs>,
10725 // [Args...],
10726 // [live variables...])
10727
10729 bool IsAnyRegCC = CC == CallingConv::AnyReg;
10730 bool HasDef = !CB.getType()->isVoidTy();
10731 SDLoc dl = getCurSDLoc();
10733
10734 // Handle immediate and symbolic callees.
10735 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
10736 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
10737 /*isTarget=*/true);
10738 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
10739 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
10740 SDLoc(SymbolicCallee),
10741 SymbolicCallee->getValueType(0));
10742
10743 // Get the real number of arguments participating in the call <numArgs>
10745 unsigned NumArgs = NArgVal->getAsZExtVal();
10746
10747 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
10748 // Intrinsics include all meta-operands up to but not including CC.
10749 unsigned NumMetaOpers = PatchPointOpers::CCPos;
10750 assert(CB.arg_size() >= NumMetaOpers + NumArgs &&
10751 "Not enough arguments provided to the patchpoint intrinsic");
10752
10753 // For AnyRegCC the arguments are lowered later on manually.
10754 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
10755 Type *ReturnTy =
10756 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType();
10757
10759 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee,
10760 ReturnTy, CB.getAttributes().getRetAttrs(), true);
10761 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
10762
10763 SDNode *CallEnd = Result.second.getNode();
10764 if (CallEnd->getOpcode() == ISD::EH_LABEL)
10765 CallEnd = CallEnd->getOperand(0).getNode();
10766 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
10767 CallEnd = CallEnd->getOperand(0).getNode();
10768
10769 /// Get a call instruction from the call sequence chain.
10770 /// Tail calls are not allowed.
10771 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&
10772 "Expected a callseq node.");
10773 SDNode *Call = CallEnd->getOperand(0).getNode();
10774 bool HasGlue = Call->getGluedNode();
10775
10776 // Replace the target specific call node with the patchable intrinsic.
10778
10779 // Push the chain.
10780 Ops.push_back(*(Call->op_begin()));
10781
10782 // Optionally, push the glue (if any).
10783 if (HasGlue)
10784 Ops.push_back(*(Call->op_end() - 1));
10785
10786 // Push the register mask info.
10787 if (HasGlue)
10788 Ops.push_back(*(Call->op_end() - 2));
10789 else
10790 Ops.push_back(*(Call->op_end() - 1));
10791
10792 // Add the <id> and <numBytes> constants.
10794 Ops.push_back(DAG.getTargetConstant(IDVal->getAsZExtVal(), dl, MVT::i64));
10796 Ops.push_back(DAG.getTargetConstant(NBytesVal->getAsZExtVal(), dl, MVT::i32));
10797
10798 // Add the callee.
10799 Ops.push_back(Callee);
10800
10801 // Adjust <numArgs> to account for any arguments that have been passed on the
10802 // stack instead.
10803 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
10804 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
10805 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
10806 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
10807
10808 // Add the calling convention
10809 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
10810
10811 // Add the arguments we omitted previously. The register allocator should
10812 // place these in any free register.
10813 if (IsAnyRegCC)
10814 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
10815 Ops.push_back(getValue(CB.getArgOperand(i)));
10816
10817 // Push the arguments from the call instruction.
10818 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
10819 Ops.append(Call->op_begin() + 2, e);
10820
10821 // Push live variables for the stack map.
10822 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this);
10823
10824 SDVTList NodeTys;
10825 if (IsAnyRegCC && HasDef) {
10826 // Create the return types based on the intrinsic definition
10828 SmallVector<EVT, 3> ValueVTs;
10829 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs);
10830 assert(ValueVTs.size() == 1 && "Expected only one return value type.");
10831
10832 // There is always a chain and a glue type at the end
10833 ValueVTs.push_back(MVT::Other);
10834 ValueVTs.push_back(MVT::Glue);
10835 NodeTys = DAG.getVTList(ValueVTs);
10836 } else
10837 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
10838
10839 // Replace the target specific call node with a PATCHPOINT node.
10840 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops);
10841
10842 // Update the NodeMap.
10843 if (HasDef) {
10844 if (IsAnyRegCC)
10845 setValue(&CB, SDValue(PPV.getNode(), 0));
10846 else
10847 setValue(&CB, Result.first);
10848 }
10849
10850 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
10851 // call sequence. Furthermore the location of the chain and glue can change
10852 // when the AnyReg calling convention is used and the intrinsic returns a
10853 // value.
10854 if (IsAnyRegCC && HasDef) {
10855 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
10856 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)};
10858 } else
10859 DAG.ReplaceAllUsesWith(Call, PPV.getNode());
10860 DAG.DeleteNode(Call);
10861
10862 // Inform the Frame Information that we have a patchpoint in this function.
10864}
10865
10866void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
10867 unsigned Intrinsic) {
10869 SDValue Op1 = getValue(I.getArgOperand(0));
10870 SDValue Op2;
10871 if (I.arg_size() > 1)
10872 Op2 = getValue(I.getArgOperand(1));
10873 SDLoc dl = getCurSDLoc();
10874 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
10875 SDValue Res;
10876 SDNodeFlags SDFlags;
10877 if (auto *FPMO = dyn_cast<FPMathOperator>(&I))
10878 SDFlags.copyFMF(*FPMO);
10879
10880 switch (Intrinsic) {
10881 case Intrinsic::vector_reduce_fadd:
10882 if (SDFlags.hasAllowReassociation())
10883 Res = DAG.getNode(ISD::FADD, dl, VT, Op1,
10884 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags),
10885 SDFlags);
10886 else
10887 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags);
10888 break;
10889 case Intrinsic::vector_reduce_fmul:
10890 if (SDFlags.hasAllowReassociation())
10891 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1,
10892 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags),
10893 SDFlags);
10894 else
10895 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags);
10896 break;
10897 case Intrinsic::vector_reduce_add:
10898 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
10899 break;
10900 case Intrinsic::vector_reduce_mul:
10901 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
10902 break;
10903 case Intrinsic::vector_reduce_and:
10904 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
10905 break;
10906 case Intrinsic::vector_reduce_or:
10907 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
10908 break;
10909 case Intrinsic::vector_reduce_xor:
10910 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
10911 break;
10912 case Intrinsic::vector_reduce_smax:
10913 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
10914 break;
10915 case Intrinsic::vector_reduce_smin:
10916 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
10917 break;
10918 case Intrinsic::vector_reduce_umax:
10919 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
10920 break;
10921 case Intrinsic::vector_reduce_umin:
10922 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
10923 break;
10924 case Intrinsic::vector_reduce_fmax:
10925 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags);
10926 break;
10927 case Intrinsic::vector_reduce_fmin:
10928 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags);
10929 break;
10930 case Intrinsic::vector_reduce_fmaximum:
10931 Res = DAG.getNode(ISD::VECREDUCE_FMAXIMUM, dl, VT, Op1, SDFlags);
10932 break;
10933 case Intrinsic::vector_reduce_fminimum:
10934 Res = DAG.getNode(ISD::VECREDUCE_FMINIMUM, dl, VT, Op1, SDFlags);
10935 break;
10936 default:
10937 llvm_unreachable("Unhandled vector reduce intrinsic");
10938 }
10939 setValue(&I, Res);
10940}
10941
10942/// Returns an AttributeList representing the attributes applied to the return
10943/// value of the given call.
10946 if (CLI.RetSExt)
10947 Attrs.push_back(Attribute::SExt);
10948 if (CLI.RetZExt)
10949 Attrs.push_back(Attribute::ZExt);
10950 if (CLI.IsInReg)
10951 Attrs.push_back(Attribute::InReg);
10952
10954 Attrs);
10955}
10956
10957/// TargetLowering::LowerCallTo - This is the default LowerCallTo
10958/// implementation, which just calls LowerCall.
10959/// FIXME: When all targets are
10960/// migrated to using LowerCall, this hook should be integrated into SDISel.
10961std::pair<SDValue, SDValue>
10963 // Handle the incoming return values from the call.
10964 CLI.Ins.clear();
10965 SmallVector<EVT, 4> RetTys;
10967 auto &DL = CLI.DAG.getDataLayout();
10968 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
10969
10970 if (CLI.IsPostTypeLegalization) {
10971 // If we are lowering a libcall after legalization, split the return type.
10972 SmallVector<EVT, 4> OldRetTys;
10973 SmallVector<TypeSize, 4> OldOffsets;
10974 RetTys.swap(OldRetTys);
10975 Offsets.swap(OldOffsets);
10976
10977 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
10978 EVT RetVT = OldRetTys[i];
10979 uint64_t Offset = OldOffsets[i];
10980 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
10981 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
10982 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
10983 RetTys.append(NumRegs, RegisterVT);
10984 for (unsigned j = 0; j != NumRegs; ++j)
10985 Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
10986 }
10987 }
10988
10990 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
10991
10992 bool CanLowerReturn =
10994 CLI.IsVarArg, Outs, CLI.RetTy->getContext(), CLI.RetTy);
10995
10996 SDValue DemoteStackSlot;
10997 int DemoteStackIdx = -100;
10998 if (!CanLowerReturn) {
10999 // FIXME: equivalent assert?
11000 // assert(!CS.hasInAllocaArgument() &&
11001 // "sret demotion is incompatible with inalloca");
11002 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
11003 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy);
11005 DemoteStackIdx =
11006 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
11007 Type *StackSlotPtrType =
11008 PointerType::get(CLI.RetTy->getContext(), DL.getAllocaAddrSpace());
11009
11010 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
11011 ArgListEntry Entry;
11012 Entry.Node = DemoteStackSlot;
11013 Entry.Ty = StackSlotPtrType;
11014 Entry.IsSExt = false;
11015 Entry.IsZExt = false;
11016 Entry.IsInReg = false;
11017 Entry.IsSRet = true;
11018 Entry.IsNest = false;
11019 Entry.IsByVal = false;
11020 Entry.IsByRef = false;
11021 Entry.IsReturned = false;
11022 Entry.IsSwiftSelf = false;
11023 Entry.IsSwiftAsync = false;
11024 Entry.IsSwiftError = false;
11025 Entry.IsCFGuardTarget = false;
11026 Entry.Alignment = Alignment;
11027 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
11028 CLI.NumFixedArgs += 1;
11029 CLI.getArgs()[0].IndirectType = CLI.RetTy;
11030 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
11031
11032 // sret demotion isn't compatible with tail-calls, since the sret argument
11033 // points into the callers stack frame.
11034 CLI.IsTailCall = false;
11035 } else {
11036 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11037 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
11038 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
11039 ISD::ArgFlagsTy Flags;
11040 if (NeedsRegBlock) {
11041 Flags.setInConsecutiveRegs();
11042 if (I == RetTys.size() - 1)
11043 Flags.setInConsecutiveRegsLast();
11044 }
11045 EVT VT = RetTys[I];
11047 CLI.CallConv, VT);
11048 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
11049 CLI.CallConv, VT);
11050 for (unsigned i = 0; i != NumRegs; ++i) {
11051 ISD::InputArg MyFlags;
11052 MyFlags.Flags = Flags;
11053 MyFlags.VT = RegisterVT;
11054 MyFlags.ArgVT = VT;
11055 MyFlags.Used = CLI.IsReturnValueUsed;
11056 if (CLI.RetTy->isPointerTy()) {
11057 MyFlags.Flags.setPointer();
11058 MyFlags.Flags.setPointerAddrSpace(
11059 cast<PointerType>(CLI.RetTy)->getAddressSpace());
11060 }
11061 if (CLI.RetSExt)
11062 MyFlags.Flags.setSExt();
11063 if (CLI.RetZExt)
11064 MyFlags.Flags.setZExt();
11065 if (CLI.IsInReg)
11066 MyFlags.Flags.setInReg();
11067 CLI.Ins.push_back(MyFlags);
11068 }
11069 }
11070 }
11071
11072 // We push in swifterror return as the last element of CLI.Ins.
11073 ArgListTy &Args = CLI.getArgs();
11074 if (supportSwiftError()) {
11075 for (const ArgListEntry &Arg : Args) {
11076 if (Arg.IsSwiftError) {
11077 ISD::InputArg MyFlags;
11078 MyFlags.VT = getPointerTy(DL);
11079 MyFlags.ArgVT = EVT(getPointerTy(DL));
11080 MyFlags.Flags.setSwiftError();
11081 CLI.Ins.push_back(MyFlags);
11082 }
11083 }
11084 }
11085
11086 // Handle all of the outgoing arguments.
11087 CLI.Outs.clear();
11088 CLI.OutVals.clear();
11089 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
11090 SmallVector<EVT, 4> ValueVTs;
11091 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
11092 // FIXME: Split arguments if CLI.IsPostTypeLegalization
11093 Type *FinalType = Args[i].Ty;
11094 if (Args[i].IsByVal)
11095 FinalType = Args[i].IndirectType;
11096 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
11097 FinalType, CLI.CallConv, CLI.IsVarArg, DL);
11098 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
11099 ++Value) {
11100 EVT VT = ValueVTs[Value];
11101 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
11102 SDValue Op = SDValue(Args[i].Node.getNode(),
11103 Args[i].Node.getResNo() + Value);
11104 ISD::ArgFlagsTy Flags;
11105
11106 // Certain targets (such as MIPS), may have a different ABI alignment
11107 // for a type depending on the context. Give the target a chance to
11108 // specify the alignment it wants.
11109 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
11110 Flags.setOrigAlign(OriginalAlignment);
11111
11112 if (Args[i].Ty->isPointerTy()) {
11113 Flags.setPointer();
11114 Flags.setPointerAddrSpace(
11115 cast<PointerType>(Args[i].Ty)->getAddressSpace());
11116 }
11117 if (Args[i].IsZExt)
11118 Flags.setZExt();
11119 if (Args[i].IsSExt)
11120 Flags.setSExt();
11121 if (Args[i].IsNoExt)
11122 Flags.setNoExt();
11123 if (Args[i].IsInReg) {
11124 // If we are using vectorcall calling convention, a structure that is
11125 // passed InReg - is surely an HVA
11127 isa<StructType>(FinalType)) {
11128 // The first value of a structure is marked
11129 if (0 == Value)
11130 Flags.setHvaStart();
11131 Flags.setHva();
11132 }
11133 // Set InReg Flag
11134 Flags.setInReg();
11135 }
11136 if (Args[i].IsSRet)
11137 Flags.setSRet();
11138 if (Args[i].IsSwiftSelf)
11139 Flags.setSwiftSelf();
11140 if (Args[i].IsSwiftAsync)
11141 Flags.setSwiftAsync();
11142 if (Args[i].IsSwiftError)
11143 Flags.setSwiftError();
11144 if (Args[i].IsCFGuardTarget)
11145 Flags.setCFGuardTarget();
11146 if (Args[i].IsByVal)
11147 Flags.setByVal();
11148 if (Args[i].IsByRef)
11149 Flags.setByRef();
11150 if (Args[i].IsPreallocated) {
11151 Flags.setPreallocated();
11152 // Set the byval flag for CCAssignFn callbacks that don't know about
11153 // preallocated. This way we can know how many bytes we should've
11154 // allocated and how many bytes a callee cleanup function will pop. If
11155 // we port preallocated to more targets, we'll have to add custom
11156 // preallocated handling in the various CC lowering callbacks.
11157 Flags.setByVal();
11158 }
11159 if (Args[i].IsInAlloca) {
11160 Flags.setInAlloca();
11161 // Set the byval flag for CCAssignFn callbacks that don't know about
11162 // inalloca. This way we can know how many bytes we should've allocated
11163 // and how many bytes a callee cleanup function will pop. If we port
11164 // inalloca to more targets, we'll have to add custom inalloca handling
11165 // in the various CC lowering callbacks.
11166 Flags.setByVal();
11167 }
11168 Align MemAlign;
11169 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) {
11170 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType);
11171 Flags.setByValSize(FrameSize);
11172
11173 // info is not there but there are cases it cannot get right.
11174 if (auto MA = Args[i].Alignment)
11175 MemAlign = *MA;
11176 else
11177 MemAlign = getByValTypeAlignment(Args[i].IndirectType, DL);
11178 } else if (auto MA = Args[i].Alignment) {
11179 MemAlign = *MA;
11180 } else {
11181 MemAlign = OriginalAlignment;
11182 }
11183 Flags.setMemAlign(MemAlign);
11184 if (Args[i].IsNest)
11185 Flags.setNest();
11186 if (NeedsRegBlock)
11187 Flags.setInConsecutiveRegs();
11188
11190 CLI.CallConv, VT);
11191 unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
11192 CLI.CallConv, VT);
11193 SmallVector<SDValue, 4> Parts(NumParts);
11194 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
11195
11196 if (Args[i].IsSExt)
11197 ExtendKind = ISD::SIGN_EXTEND;
11198 else if (Args[i].IsZExt)
11199 ExtendKind = ISD::ZERO_EXTEND;
11200
11201 // Conservatively only handle 'returned' on non-vectors that can be lowered,
11202 // for now.
11203 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
11205 assert((CLI.RetTy == Args[i].Ty ||
11206 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
11208 Args[i].Ty->getPointerAddressSpace())) &&
11209 RetTys.size() == NumValues && "unexpected use of 'returned'");
11210 // Before passing 'returned' to the target lowering code, ensure that
11211 // either the register MVT and the actual EVT are the same size or that
11212 // the return value and argument are extended in the same way; in these
11213 // cases it's safe to pass the argument register value unchanged as the
11214 // return register value (although it's at the target's option whether
11215 // to do so)
11216 // TODO: allow code generation to take advantage of partially preserved
11217 // registers rather than clobbering the entire register when the
11218 // parameter extension method is not compatible with the return
11219 // extension method
11220 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
11221 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
11222 CLI.RetZExt == Args[i].IsZExt))
11223 Flags.setReturned();
11224 }
11225
11226 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB,
11227 CLI.CallConv, ExtendKind);
11228
11229 for (unsigned j = 0; j != NumParts; ++j) {
11230 // if it isn't first piece, alignment must be 1
11231 // For scalable vectors the scalable part is currently handled
11232 // by individual targets, so we just use the known minimum size here.
11233 ISD::OutputArg MyFlags(
11234 Flags, Parts[j].getValueType().getSimpleVT(), VT,
11235 i < CLI.NumFixedArgs, i,
11236 j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
11237 if (NumParts > 1 && j == 0)
11238 MyFlags.Flags.setSplit();
11239 else if (j != 0) {
11240 MyFlags.Flags.setOrigAlign(Align(1));
11241 if (j == NumParts - 1)
11242 MyFlags.Flags.setSplitEnd();
11243 }
11244
11245 CLI.Outs.push_back(MyFlags);
11246 CLI.OutVals.push_back(Parts[j]);
11247 }
11248
11249 if (NeedsRegBlock && Value == NumValues - 1)
11250 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
11251 }
11252 }
11253
11255 CLI.Chain = LowerCall(CLI, InVals);
11256
11257 // Update CLI.InVals to use outside of this function.
11258 CLI.InVals = InVals;
11259
11260 // Verify that the target's LowerCall behaved as expected.
11261 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&
11262 "LowerCall didn't return a valid chain!");
11263 assert((!CLI.IsTailCall || InVals.empty()) &&
11264 "LowerCall emitted a return value for a tail call!");
11265 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&
11266 "LowerCall didn't emit the correct number of values!");
11267
11268 // For a tail call, the return value is merely live-out and there aren't
11269 // any nodes in the DAG representing it. Return a special value to
11270 // indicate that a tail call has been emitted and no more Instructions
11271 // should be processed in the current block.
11272 if (CLI.IsTailCall) {
11273 CLI.DAG.setRoot(CLI.Chain);
11274 return std::make_pair(SDValue(), SDValue());
11275 }
11276
11277#ifndef NDEBUG
11278 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
11279 assert(InVals[i].getNode() && "LowerCall emitted a null value!");
11280 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&
11281 "LowerCall emitted a value with the wrong type!");
11282 }
11283#endif
11284
11285 SmallVector<SDValue, 4> ReturnValues;
11286 if (!CanLowerReturn) {
11287 // The instruction result is the result of loading from the
11288 // hidden sret parameter.
11289 MVT PtrVT = getPointerTy(DL, DL.getAllocaAddrSpace());
11290
11291 unsigned NumValues = RetTys.size();
11292 ReturnValues.resize(NumValues);
11293 SmallVector<SDValue, 4> Chains(NumValues);
11294
11295 // An aggregate return value cannot wrap around the address space, so
11296 // offsets to its parts don't wrap either.
11298 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx);
11299 for (unsigned i = 0; i < NumValues; ++i) {
11300 SDValue Add =
11301 CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
11302 CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT),
11304 SDValue L = CLI.DAG.getLoad(
11305 RetTys[i], CLI.DL, CLI.Chain, Add,
11307 DemoteStackIdx, Offsets[i]),
11308 HiddenSRetAlign);
11309 ReturnValues[i] = L;
11310 Chains[i] = L.getValue(1);
11311 }
11312
11313 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
11314 } else {
11315 // Collect the legal value parts into potentially illegal values
11316 // that correspond to the original function's return values.
11317 std::optional<ISD::NodeType> AssertOp;
11318 if (CLI.RetSExt)
11319 AssertOp = ISD::AssertSext;
11320 else if (CLI.RetZExt)
11321 AssertOp = ISD::AssertZext;
11322 unsigned CurReg = 0;
11323 for (EVT VT : RetTys) {
11325 CLI.CallConv, VT);
11326 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
11327 CLI.CallConv, VT);
11328
11329 ReturnValues.push_back(getCopyFromParts(
11330 CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
11331 CLI.Chain, CLI.CallConv, AssertOp));
11332 CurReg += NumRegs;
11333 }
11334
11335 // For a function returning void, there is no return value. We can't create
11336 // such a node, so we just return a null return value in that case. In
11337 // that case, nothing will actually look at the value.
11338 if (ReturnValues.empty())
11339 return std::make_pair(SDValue(), CLI.Chain);
11340 }
11341
11342 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
11343 CLI.DAG.getVTList(RetTys), ReturnValues);
11344 return std::make_pair(Res, CLI.Chain);
11345}
11346
11347/// Places new result values for the node in Results (their number
11348/// and types must exactly match those of the original return values of
11349/// the node), or leaves Results empty, which indicates that the node is not
11350/// to be custom lowered after all.
11353 SelectionDAG &DAG) const {
11354 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
11355
11356 if (!Res.getNode())
11357 return;
11358
11359 // If the original node has one result, take the return value from
11360 // LowerOperation as is. It might not be result number 0.
11361 if (N->getNumValues() == 1) {
11362 Results.push_back(Res);
11363 return;
11364 }
11365
11366 // If the original node has multiple results, then the return node should
11367 // have the same number of results.
11368 assert((N->getNumValues() == Res->getNumValues()) &&
11369 "Lowering returned the wrong number of results!");
11370
11371 // Places new result values base on N result number.
11372 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
11373 Results.push_back(Res.getValue(I));
11374}
11375
11377 llvm_unreachable("LowerOperation not implemented for this target!");
11378}
11379
11381 unsigned Reg,
11382 ISD::NodeType ExtendType) {
11384 assert((Op.getOpcode() != ISD::CopyFromReg ||
11385 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&
11386 "Copy from a reg to the same reg!");
11387 assert(!Register::isPhysicalRegister(Reg) && "Is a physreg");
11388
11390 // If this is an InlineAsm we have to match the registers required, not the
11391 // notional registers required by the type.
11392
11393 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
11394 std::nullopt); // This is not an ABI copy.
11395 SDValue Chain = DAG.getEntryNode();
11396
11397 if (ExtendType == ISD::ANY_EXTEND) {
11398 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V);
11399 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end())
11400 ExtendType = PreferredExtendIt->second;
11401 }
11402 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
11403 PendingExports.push_back(Chain);
11404}
11405
11407
11408/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
11409/// entry block, return true. This includes arguments used by switches, since
11410/// the switch may expand into multiple basic blocks.
11411static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
11412 // With FastISel active, we may be splitting blocks, so force creation
11413 // of virtual registers for all non-dead arguments.
11414 if (FastISel)
11415 return A->use_empty();
11416
11417 const BasicBlock &Entry = A->getParent()->front();
11418 for (const User *U : A->users())
11419 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
11420 return false; // Use not in entry block.
11421
11422 return true;
11423}
11424
11426 DenseMap<const Argument *,
11427 std::pair<const AllocaInst *, const StoreInst *>>;
11428
11429/// Scan the entry block of the function in FuncInfo for arguments that look
11430/// like copies into a local alloca. Record any copied arguments in
11431/// ArgCopyElisionCandidates.
11432static void
11434 FunctionLoweringInfo *FuncInfo,
11435 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
11436 // Record the state of every static alloca used in the entry block. Argument
11437 // allocas are all used in the entry block, so we need approximately as many
11438 // entries as we have arguments.
11439 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
11441 unsigned NumArgs = FuncInfo->Fn->arg_size();
11442 StaticAllocas.reserve(NumArgs * 2);
11443
11444 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
11445 if (!V)
11446 return nullptr;
11447 V = V->stripPointerCasts();
11448 const auto *AI = dyn_cast<AllocaInst>(V);
11449 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
11450 return nullptr;
11451 auto Iter = StaticAllocas.insert({AI, Unknown});
11452 return &Iter.first->second;
11453 };
11454
11455 // Look for stores of arguments to static allocas. Look through bitcasts and
11456 // GEPs to handle type coercions, as long as the alloca is fully initialized
11457 // by the store. Any non-store use of an alloca escapes it and any subsequent
11458 // unanalyzed store might write it.
11459 // FIXME: Handle structs initialized with multiple stores.
11460 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
11461 // Look for stores, and handle non-store uses conservatively.
11462 const auto *SI = dyn_cast<StoreInst>(&I);
11463 if (!SI) {
11464 // We will look through cast uses, so ignore them completely.
11465 if (I.isCast())
11466 continue;
11467 // Ignore debug info and pseudo op intrinsics, they don't escape or store
11468 // to allocas.
11469 if (I.isDebugOrPseudoInst())
11470 continue;
11471 // This is an unknown instruction. Assume it escapes or writes to all
11472 // static alloca operands.
11473 for (const Use &U : I.operands()) {
11474 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
11475 *Info = StaticAllocaInfo::Clobbered;
11476 }
11477 continue;
11478 }
11479
11480 // If the stored value is a static alloca, mark it as escaped.
11481 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
11482 *Info = StaticAllocaInfo::Clobbered;
11483
11484 // Check if the destination is a static alloca.
11485 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
11486 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
11487 if (!Info)
11488 continue;
11489 const AllocaInst *AI = cast<AllocaInst>(Dst);
11490
11491 // Skip allocas that have been initialized or clobbered.
11492 if (*Info != StaticAllocaInfo::Unknown)
11493 continue;
11494
11495 // Check if the stored value is an argument, and that this store fully
11496 // initializes the alloca.
11497 // If the argument type has padding bits we can't directly forward a pointer
11498 // as the upper bits may contain garbage.
11499 // Don't elide copies from the same argument twice.
11500 const Value *Val = SI->getValueOperand()->stripPointerCasts();
11501 const auto *Arg = dyn_cast<Argument>(Val);
11502 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() ||
11503 Arg->getType()->isEmptyTy() ||
11504 DL.getTypeStoreSize(Arg->getType()) !=
11505 DL.getTypeAllocSize(AI->getAllocatedType()) ||
11506 !DL.typeSizeEqualsStoreSize(Arg->getType()) ||
11507 ArgCopyElisionCandidates.count(Arg)) {
11508 *Info = StaticAllocaInfo::Clobbered;
11509 continue;
11510 }
11511
11512 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI
11513 << '\n');
11514
11515 // Mark this alloca and store for argument copy elision.
11516 *Info = StaticAllocaInfo::Elidable;
11517 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
11518
11519 // Stop scanning if we've seen all arguments. This will happen early in -O0
11520 // builds, which is useful, because -O0 builds have large entry blocks and
11521 // many allocas.
11522 if (ArgCopyElisionCandidates.size() == NumArgs)
11523 break;
11524 }
11525}
11526
11527/// Try to elide argument copies from memory into a local alloca. Succeeds if
11528/// ArgVal is a load from a suitable fixed stack object.
11531 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
11532 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
11533 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
11534 ArrayRef<SDValue> ArgVals, bool &ArgHasUses) {
11535 // Check if this is a load from a fixed stack object.
11536 auto *LNode = dyn_cast<LoadSDNode>(ArgVals[0]);
11537 if (!LNode)
11538 return;
11539 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
11540 if (!FINode)
11541 return;
11542
11543 // Check that the fixed stack object is the right size and alignment.
11544 // Look at the alignment that the user wrote on the alloca instead of looking
11545 // at the stack object.
11546 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
11547 assert(ArgCopyIter != ArgCopyElisionCandidates.end());
11548 const AllocaInst *AI = ArgCopyIter->second.first;
11549 int FixedIndex = FINode->getIndex();
11550 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI];
11551 int OldIndex = AllocaIndex;
11552 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo();
11553 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
11554 LLVM_DEBUG(
11555 dbgs() << " argument copy elision failed due to bad fixed stack "
11556 "object size\n");
11557 return;
11558 }
11559 Align RequiredAlignment = AI->getAlign();
11560 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) {
11561 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "
11562 "greater than stack argument alignment ("
11563 << DebugStr(RequiredAlignment) << " vs "
11564 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n");
11565 return;
11566 }
11567
11568 // Perform the elision. Delete the old stack object and replace its only use
11569 // in the variable info map. Mark the stack object as mutable and aliased.
11570 LLVM_DEBUG({
11571 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'
11572 << " Replacing frame index " << OldIndex << " with " << FixedIndex
11573 << '\n';
11574 });
11575 MFI.RemoveStackObject(OldIndex);
11576 MFI.setIsImmutableObjectIndex(FixedIndex, false);
11577 MFI.setIsAliasedObjectIndex(FixedIndex, true);
11578 AllocaIndex = FixedIndex;
11579 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
11580 for (SDValue ArgVal : ArgVals)
11581 Chains.push_back(ArgVal.getValue(1));
11582
11583 // Avoid emitting code for the store implementing the copy.
11584 const StoreInst *SI = ArgCopyIter->second.second;
11585 ElidedArgCopyInstrs.insert(SI);
11586
11587 // Check for uses of the argument again so that we can avoid exporting ArgVal
11588 // if it is't used by anything other than the store.
11589 for (const Value *U : Arg.users()) {
11590 if (U != SI) {
11591 ArgHasUses = true;
11592 break;
11593 }
11594 }
11595}
11596
11597void SelectionDAGISel::LowerArguments(const Function &F) {
11598 SelectionDAG &DAG = SDB->DAG;
11599 SDLoc dl = SDB->getCurSDLoc();
11600 const DataLayout &DL = DAG.getDataLayout();
11602
11603 // In Naked functions we aren't going to save any registers.
11604 if (F.hasFnAttribute(Attribute::Naked))
11605 return;
11606
11607 if (!FuncInfo->CanLowerReturn) {
11608 // Put in an sret pointer parameter before all the other parameters.
11609 MVT ValueVT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
11610
11612 Flags.setSRet();
11613 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVT);
11614 ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, true,
11616 Ins.push_back(RetArg);
11617 }
11618
11619 // Look for stores of arguments to static allocas. Mark such arguments with a
11620 // flag to ask the target to give us the memory location of that argument if
11621 // available.
11622 ArgCopyElisionMapTy ArgCopyElisionCandidates;
11624 ArgCopyElisionCandidates);
11625
11626 // Set up the incoming argument description vector.
11627 for (const Argument &Arg : F.args()) {
11628 unsigned ArgNo = Arg.getArgNo();
11629 SmallVector<EVT, 4> ValueVTs;
11630 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11631 bool isArgValueUsed = !Arg.use_empty();
11632 unsigned PartBase = 0;
11633 Type *FinalType = Arg.getType();
11634 if (Arg.hasAttribute(Attribute::ByVal))
11635 FinalType = Arg.getParamByValType();
11636 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
11637 FinalType, F.getCallingConv(), F.isVarArg(), DL);
11638 for (unsigned Value = 0, NumValues = ValueVTs.size();
11639 Value != NumValues; ++Value) {
11640 EVT VT = ValueVTs[Value];
11641 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
11643
11644
11645 if (Arg.getType()->isPointerTy()) {
11646 Flags.setPointer();
11647 Flags.setPointerAddrSpace(
11648 cast<PointerType>(Arg.getType())->getAddressSpace());
11649 }
11650 if (Arg.hasAttribute(Attribute::ZExt))
11651 Flags.setZExt();
11652 if (Arg.hasAttribute(Attribute::SExt))
11653 Flags.setSExt();
11654 if (Arg.hasAttribute(Attribute::InReg)) {
11655 // If we are using vectorcall calling convention, a structure that is
11656 // passed InReg - is surely an HVA
11657 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
11658 isa<StructType>(Arg.getType())) {
11659 // The first value of a structure is marked
11660 if (0 == Value)
11661 Flags.setHvaStart();
11662 Flags.setHva();
11663 }
11664 // Set InReg Flag
11665 Flags.setInReg();
11666 }
11667 if (Arg.hasAttribute(Attribute::StructRet))
11668 Flags.setSRet();
11669 if (Arg.hasAttribute(Attribute::SwiftSelf))
11670 Flags.setSwiftSelf();
11671 if (Arg.hasAttribute(Attribute::SwiftAsync))
11672 Flags.setSwiftAsync();
11673 if (Arg.hasAttribute(Attribute::SwiftError))
11674 Flags.setSwiftError();
11675 if (Arg.hasAttribute(Attribute::ByVal))
11676 Flags.setByVal();
11677 if (Arg.hasAttribute(Attribute::ByRef))
11678 Flags.setByRef();
11679 if (Arg.hasAttribute(Attribute::InAlloca)) {
11680 Flags.setInAlloca();
11681 // Set the byval flag for CCAssignFn callbacks that don't know about
11682 // inalloca. This way we can know how many bytes we should've allocated
11683 // and how many bytes a callee cleanup function will pop. If we port
11684 // inalloca to more targets, we'll have to add custom inalloca handling
11685 // in the various CC lowering callbacks.
11686 Flags.setByVal();
11687 }
11688 if (Arg.hasAttribute(Attribute::Preallocated)) {
11689 Flags.setPreallocated();
11690 // Set the byval flag for CCAssignFn callbacks that don't know about
11691 // preallocated. This way we can know how many bytes we should've
11692 // allocated and how many bytes a callee cleanup function will pop. If
11693 // we port preallocated to more targets, we'll have to add custom
11694 // preallocated handling in the various CC lowering callbacks.
11695 Flags.setByVal();
11696 }
11697
11698 // Certain targets (such as MIPS), may have a different ABI alignment
11699 // for a type depending on the context. Give the target a chance to
11700 // specify the alignment it wants.
11701 const Align OriginalAlignment(
11703 Flags.setOrigAlign(OriginalAlignment);
11704
11705 Align MemAlign;
11706 Type *ArgMemTy = nullptr;
11707 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() ||
11708 Flags.isByRef()) {
11709 if (!ArgMemTy)
11710 ArgMemTy = Arg.getPointeeInMemoryValueType();
11711
11712 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy);
11713
11714 // For in-memory arguments, size and alignment should be passed from FE.
11715 // BE will guess if this info is not there but there are cases it cannot
11716 // get right.
11717 if (auto ParamAlign = Arg.getParamStackAlign())
11718 MemAlign = *ParamAlign;
11719 else if ((ParamAlign = Arg.getParamAlign()))
11720 MemAlign = *ParamAlign;
11721 else
11722 MemAlign = TLI->getByValTypeAlignment(ArgMemTy, DL);
11723 if (Flags.isByRef())
11724 Flags.setByRefSize(MemSize);
11725 else
11726 Flags.setByValSize(MemSize);
11727 } else if (auto ParamAlign = Arg.getParamStackAlign()) {
11728 MemAlign = *ParamAlign;
11729 } else {
11730 MemAlign = OriginalAlignment;
11731 }
11732 Flags.setMemAlign(MemAlign);
11733
11734 if (Arg.hasAttribute(Attribute::Nest))
11735 Flags.setNest();
11736 if (NeedsRegBlock)
11737 Flags.setInConsecutiveRegs();
11738 if (ArgCopyElisionCandidates.count(&Arg))
11739 Flags.setCopyElisionCandidate();
11740 if (Arg.hasAttribute(Attribute::Returned))
11741 Flags.setReturned();
11742
11744 *CurDAG->getContext(), F.getCallingConv(), VT);
11745 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
11746 *CurDAG->getContext(), F.getCallingConv(), VT);
11747 for (unsigned i = 0; i != NumRegs; ++i) {
11748 // For scalable vectors, use the minimum size; individual targets
11749 // are responsible for handling scalable vector arguments and
11750 // return values.
11751 ISD::InputArg MyFlags(
11752 Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
11753 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
11754 if (NumRegs > 1 && i == 0)
11755 MyFlags.Flags.setSplit();
11756 // if it isn't first piece, alignment must be 1
11757 else if (i > 0) {
11758 MyFlags.Flags.setOrigAlign(Align(1));
11759 if (i == NumRegs - 1)
11760 MyFlags.Flags.setSplitEnd();
11761 }
11762 Ins.push_back(MyFlags);
11763 }
11764 if (NeedsRegBlock && Value == NumValues - 1)
11765 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
11766 PartBase += VT.getStoreSize().getKnownMinValue();
11767 }
11768 }
11769
11770 // Call the target to set up the argument values.
11772 SDValue NewRoot = TLI->LowerFormalArguments(
11773 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
11774
11775 // Verify that the target's LowerFormalArguments behaved as expected.
11776 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&
11777 "LowerFormalArguments didn't return a valid chain!");
11778 assert(InVals.size() == Ins.size() &&
11779 "LowerFormalArguments didn't emit the correct number of values!");
11780 LLVM_DEBUG({
11781 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
11782 assert(InVals[i].getNode() &&
11783 "LowerFormalArguments emitted a null value!");
11784 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&
11785 "LowerFormalArguments emitted a value with the wrong type!");
11786 }
11787 });
11788
11789 // Update the DAG with the new chain value resulting from argument lowering.
11790 DAG.setRoot(NewRoot);
11791
11792 // Set up the argument values.
11793 unsigned i = 0;
11794 if (!FuncInfo->CanLowerReturn) {
11795 // Create a virtual register for the sret pointer, and put in a copy
11796 // from the sret argument into it.
11797 MVT VT = TLI->getPointerTy(DL, DL.getAllocaAddrSpace());
11798 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
11799 std::optional<ISD::NodeType> AssertOp;
11800 SDValue ArgValue =
11801 getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot,
11802 F.getCallingConv(), AssertOp);
11803
11804 MachineFunction& MF = SDB->DAG.getMachineFunction();
11806 Register SRetReg =
11807 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
11808 FuncInfo->DemoteRegister = SRetReg;
11809 NewRoot =
11810 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
11811 DAG.setRoot(NewRoot);
11812
11813 // i indexes lowered arguments. Bump it past the hidden sret argument.
11814 ++i;
11815 }
11816
11818 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
11819 for (const Argument &Arg : F.args()) {
11820 SmallVector<SDValue, 4> ArgValues;
11821 SmallVector<EVT, 4> ValueVTs;
11822 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
11823 unsigned NumValues = ValueVTs.size();
11824 if (NumValues == 0)
11825 continue;
11826
11827 bool ArgHasUses = !Arg.use_empty();
11828
11829 // Elide the copying store if the target loaded this argument from a
11830 // suitable fixed stack object.
11831 if (Ins[i].Flags.isCopyElisionCandidate()) {
11832 unsigned NumParts = 0;
11833 for (EVT VT : ValueVTs)
11835 F.getCallingConv(), VT);
11836
11837 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
11838 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
11839 ArrayRef(&InVals[i], NumParts), ArgHasUses);
11840 }
11841
11842 // If this argument is unused then remember its value. It is used to generate
11843 // debugging information.
11844 bool isSwiftErrorArg =
11846 Arg.hasAttribute(Attribute::SwiftError);
11847 if (!ArgHasUses && !isSwiftErrorArg) {
11848 SDB->setUnusedArgValue(&Arg, InVals[i]);
11849
11850 // Also remember any frame index for use in FastISel.
11851 if (FrameIndexSDNode *FI =
11852 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
11853 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11854 }
11855
11856 for (unsigned Val = 0; Val != NumValues; ++Val) {
11857 EVT VT = ValueVTs[Val];
11859 F.getCallingConv(), VT);
11860 unsigned NumParts = TLI->getNumRegistersForCallingConv(
11861 *CurDAG->getContext(), F.getCallingConv(), VT);
11862
11863 // Even an apparent 'unused' swifterror argument needs to be returned. So
11864 // we do generate a copy for it that can be used on return from the
11865 // function.
11866 if (ArgHasUses || isSwiftErrorArg) {
11867 std::optional<ISD::NodeType> AssertOp;
11868 if (Arg.hasAttribute(Attribute::SExt))
11869 AssertOp = ISD::AssertSext;
11870 else if (Arg.hasAttribute(Attribute::ZExt))
11871 AssertOp = ISD::AssertZext;
11872
11873 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
11874 PartVT, VT, nullptr, NewRoot,
11875 F.getCallingConv(), AssertOp));
11876 }
11877
11878 i += NumParts;
11879 }
11880
11881 // We don't need to do anything else for unused arguments.
11882 if (ArgValues.empty())
11883 continue;
11884
11885 // Note down frame index.
11886 if (FrameIndexSDNode *FI =
11887 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
11888 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11889
11890 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues),
11891 SDB->getCurSDLoc());
11892
11893 SDB->setValue(&Arg, Res);
11895 // We want to associate the argument with the frame index, among
11896 // involved operands, that correspond to the lowest address. The
11897 // getCopyFromParts function, called earlier, is swapping the order of
11898 // the operands to BUILD_PAIR depending on endianness. The result of
11899 // that swapping is that the least significant bits of the argument will
11900 // be in the first operand of the BUILD_PAIR node, and the most
11901 // significant bits will be in the second operand.
11902 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
11903 if (LoadSDNode *LNode =
11904 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
11905 if (FrameIndexSDNode *FI =
11906 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
11907 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
11908 }
11909
11910 // Analyses past this point are naive and don't expect an assertion.
11911 if (Res.getOpcode() == ISD::AssertZext)
11912 Res = Res.getOperand(0);
11913
11914 // Update the SwiftErrorVRegDefMap.
11915 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
11916 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11917 if (Reg.isVirtual())
11919 Reg);
11920 }
11921
11922 // If this argument is live outside of the entry block, insert a copy from
11923 // wherever we got it to the vreg that other BB's will reference it as.
11924 if (Res.getOpcode() == ISD::CopyFromReg) {
11925 // If we can, though, try to skip creating an unnecessary vreg.
11926 // FIXME: This isn't very clean... it would be nice to make this more
11927 // general.
11928 Register Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
11929 if (Reg.isVirtual()) {
11930 FuncInfo->ValueMap[&Arg] = Reg;
11931 continue;
11932 }
11933 }
11935 FuncInfo->InitializeRegForValue(&Arg);
11936 SDB->CopyToExportRegsIfNeeded(&Arg);
11937 }
11938 }
11939
11940 if (!Chains.empty()) {
11941 Chains.push_back(NewRoot);
11942 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
11943 }
11944
11945 DAG.setRoot(NewRoot);
11946
11947 assert(i == InVals.size() && "Argument register count mismatch!");
11948
11949 // If any argument copy elisions occurred and we have debug info, update the
11950 // stale frame indices used in the dbg.declare variable info table.
11951 if (!ArgCopyElisionFrameIndexMap.empty()) {
11954 auto I = ArgCopyElisionFrameIndexMap.find(VI.getStackSlot());
11955 if (I != ArgCopyElisionFrameIndexMap.end())
11956 VI.updateStackSlot(I->second);
11957 }
11958 }
11959
11960 // Finally, if the target has anything special to do, allow it to do so.
11962}
11963
11964/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
11965/// ensure constants are generated when needed. Remember the virtual registers
11966/// that need to be added to the Machine PHI nodes as input. We cannot just
11967/// directly add them, because expansion might result in multiple MBB's for one
11968/// BB. As such, the start of the BB might correspond to a different MBB than
11969/// the end.
11970void
11971SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
11973
11975
11976 // Check PHI nodes in successors that expect a value to be available from this
11977 // block.
11978 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) {
11979 if (!isa<PHINode>(SuccBB->begin())) continue;
11980 MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB);
11981
11982 // If this terminator has multiple identical successors (common for
11983 // switches), only handle each succ once.
11984 if (!SuccsHandled.insert(SuccMBB).second)
11985 continue;
11986
11988
11989 // At this point we know that there is a 1-1 correspondence between LLVM PHI
11990 // nodes and Machine PHI nodes, but the incoming operands have not been
11991 // emitted yet.
11992 for (const PHINode &PN : SuccBB->phis()) {
11993 // Ignore dead phi's.
11994 if (PN.use_empty())
11995 continue;
11996
11997 // Skip empty types
11998 if (PN.getType()->isEmptyTy())
11999 continue;
12000
12001 unsigned Reg;
12002 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
12003
12004 if (const auto *C = dyn_cast<Constant>(PHIOp)) {
12005 unsigned &RegOut = ConstantsOut[C];
12006 if (RegOut == 0) {
12007 RegOut = FuncInfo.CreateRegs(C);
12008 // We need to zero/sign extend ConstantInt phi operands to match
12009 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
12010 ISD::NodeType ExtendType = ISD::ANY_EXTEND;
12011 if (auto *CI = dyn_cast<ConstantInt>(C))
12012 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND
12014 CopyValueToVirtualRegister(C, RegOut, ExtendType);
12015 }
12016 Reg = RegOut;
12017 } else {
12019 FuncInfo.ValueMap.find(PHIOp);
12020 if (I != FuncInfo.ValueMap.end())
12021 Reg = I->second;
12022 else {
12023 assert(isa<AllocaInst>(PHIOp) &&
12024 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
12025 "Didn't codegen value into a register!??");
12026 Reg = FuncInfo.CreateRegs(PHIOp);
12027 CopyValueToVirtualRegister(PHIOp, Reg);
12028 }
12029 }
12030
12031 // Remember that this register needs to added to the machine PHI node as
12032 // the input for this MBB.
12033 SmallVector<EVT, 4> ValueVTs;
12034 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
12035 for (EVT VT : ValueVTs) {
12036 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
12037 for (unsigned i = 0; i != NumRegisters; ++i)
12038 FuncInfo.PHINodesToUpdate.push_back(
12039 std::make_pair(&*MBBI++, Reg + i));
12040 Reg += NumRegisters;
12041 }
12042 }
12043 }
12044
12045 ConstantsOut.clear();
12046}
12047
12048MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
12050 if (++I == FuncInfo.MF->end())
12051 return nullptr;
12052 return &*I;
12053}
12054
12055/// During lowering new call nodes can be created (such as memset, etc.).
12056/// Those will become new roots of the current DAG, but complications arise
12057/// when they are tail calls. In such cases, the call lowering will update
12058/// the root, but the builder still needs to know that a tail call has been
12059/// lowered in order to avoid generating an additional return.
12060void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
12061 // If the node is null, we do have a tail call.
12062 if (MaybeTC.getNode() != nullptr)
12063 DAG.setRoot(MaybeTC);
12064 else
12065 HasTailCall = true;
12066}
12067
12068void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
12069 MachineBasicBlock *SwitchMBB,
12070 MachineBasicBlock *DefaultMBB) {
12071 MachineFunction *CurMF = FuncInfo.MF;
12072 MachineBasicBlock *NextMBB = nullptr;
12074 if (++BBI != FuncInfo.MF->end())
12075 NextMBB = &*BBI;
12076
12077 unsigned Size = W.LastCluster - W.FirstCluster + 1;
12078
12080
12081 if (Size == 2 && W.MBB == SwitchMBB) {
12082 // If any two of the cases has the same destination, and if one value
12083 // is the same as the other, but has one bit unset that the other has set,
12084 // use bit manipulation to do two compares at once. For example:
12085 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
12086 // TODO: This could be extended to merge any 2 cases in switches with 3
12087 // cases.
12088 // TODO: Handle cases where W.CaseBB != SwitchBB.
12089 CaseCluster &Small = *W.FirstCluster;
12090 CaseCluster &Big = *W.LastCluster;
12091
12092 if (Small.Low == Small.High && Big.Low == Big.High &&
12093 Small.MBB == Big.MBB) {
12094 const APInt &SmallValue = Small.Low->getValue();
12095 const APInt &BigValue = Big.Low->getValue();
12096
12097 // Check that there is only one bit different.
12098 APInt CommonBit = BigValue ^ SmallValue;
12099 if (CommonBit.isPowerOf2()) {
12100 SDValue CondLHS = getValue(Cond);
12101 EVT VT = CondLHS.getValueType();
12102 SDLoc DL = getCurSDLoc();
12103
12104 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
12105 DAG.getConstant(CommonBit, DL, VT));
12107 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
12108 ISD::SETEQ);
12109
12110 // Update successor info.
12111 // Both Small and Big will jump to Small.BB, so we sum up the
12112 // probabilities.
12113 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
12114 if (BPI)
12115 addSuccessorWithProb(
12116 SwitchMBB, DefaultMBB,
12117 // The default destination is the first successor in IR.
12118 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
12119 else
12120 addSuccessorWithProb(SwitchMBB, DefaultMBB);
12121
12122 // Insert the true branch.
12123 SDValue BrCond =
12124 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
12125 DAG.getBasicBlock(Small.MBB));
12126 // Insert the false branch.
12127 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
12128 DAG.getBasicBlock(DefaultMBB));
12129
12130 DAG.setRoot(BrCond);
12131 return;
12132 }
12133 }
12134 }
12135
12136 if (TM.getOptLevel() != CodeGenOptLevel::None) {
12137 // Here, we order cases by probability so the most likely case will be
12138 // checked first. However, two clusters can have the same probability in
12139 // which case their relative ordering is non-deterministic. So we use Low
12140 // as a tie-breaker as clusters are guaranteed to never overlap.
12141 llvm::sort(W.FirstCluster, W.LastCluster + 1,
12142 [](const CaseCluster &a, const CaseCluster &b) {
12143 return a.Prob != b.Prob ?
12144 a.Prob > b.Prob :
12145 a.Low->getValue().slt(b.Low->getValue());
12146 });
12147
12148 // Rearrange the case blocks so that the last one falls through if possible
12149 // without changing the order of probabilities.
12150 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
12151 --I;
12152 if (I->Prob > W.LastCluster->Prob)
12153 break;
12154 if (I->Kind == CC_Range && I->MBB == NextMBB) {
12155 std::swap(*I, *W.LastCluster);
12156 break;
12157 }
12158 }
12159 }
12160
12161 // Compute total probability.
12162 BranchProbability DefaultProb = W.DefaultProb;
12163 BranchProbability UnhandledProbs = DefaultProb;
12164 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
12165 UnhandledProbs += I->Prob;
12166
12167 MachineBasicBlock *CurMBB = W.MBB;
12168 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
12169 bool FallthroughUnreachable = false;
12170 MachineBasicBlock *Fallthrough;
12171 if (I == W.LastCluster) {
12172 // For the last cluster, fall through to the default destination.
12173 Fallthrough = DefaultMBB;
12174 FallthroughUnreachable = isa<UnreachableInst>(
12175 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
12176 } else {
12177 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
12178 CurMF->insert(BBI, Fallthrough);
12179 // Put Cond in a virtual register to make it available from the new blocks.
12181 }
12182 UnhandledProbs -= I->Prob;
12183
12184 switch (I->Kind) {
12185 case CC_JumpTable: {
12186 // FIXME: Optimize away range check based on pivot comparisons.
12187 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
12188 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
12189
12190 // The jump block hasn't been inserted yet; insert it here.
12191 MachineBasicBlock *JumpMBB = JT->MBB;
12192 CurMF->insert(BBI, JumpMBB);
12193
12194 auto JumpProb = I->Prob;
12195 auto FallthroughProb = UnhandledProbs;
12196
12197 // If the default statement is a target of the jump table, we evenly
12198 // distribute the default probability to successors of CurMBB. Also
12199 // update the probability on the edge from JumpMBB to Fallthrough.
12200 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
12201 SE = JumpMBB->succ_end();
12202 SI != SE; ++SI) {
12203 if (*SI == DefaultMBB) {
12204 JumpProb += DefaultProb / 2;
12205 FallthroughProb -= DefaultProb / 2;
12206 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
12207 JumpMBB->normalizeSuccProbs();
12208 break;
12209 }
12210 }
12211
12212 // If the default clause is unreachable, propagate that knowledge into
12213 // JTH->FallthroughUnreachable which will use it to suppress the range
12214 // check.
12215 //
12216 // However, don't do this if we're doing branch target enforcement,
12217 // because a table branch _without_ a range check can be a tempting JOP
12218 // gadget - out-of-bounds inputs that are impossible in correct
12219 // execution become possible again if an attacker can influence the
12220 // control flow. So if an attacker doesn't already have a BTI bypass
12221 // available, we don't want them to be able to get one out of this
12222 // table branch.
12223 if (FallthroughUnreachable) {
12224 Function &CurFunc = CurMF->getFunction();
12225 if (!CurFunc.hasFnAttribute("branch-target-enforcement"))
12226 JTH->FallthroughUnreachable = true;
12227 }
12228
12229 if (!JTH->FallthroughUnreachable)
12230 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
12231 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
12232 CurMBB->normalizeSuccProbs();
12233
12234 // The jump table header will be inserted in our current block, do the
12235 // range check, and fall through to our fallthrough block.
12236 JTH->HeaderBB = CurMBB;
12237 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
12238
12239 // If we're in the right place, emit the jump table header right now.
12240 if (CurMBB == SwitchMBB) {
12241 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
12242 JTH->Emitted = true;
12243 }
12244 break;
12245 }
12246 case CC_BitTests: {
12247 // FIXME: Optimize away range check based on pivot comparisons.
12248 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
12249
12250 // The bit test blocks haven't been inserted yet; insert them here.
12251 for (BitTestCase &BTC : BTB->Cases)
12252 CurMF->insert(BBI, BTC.ThisBB);
12253
12254 // Fill in fields of the BitTestBlock.
12255 BTB->Parent = CurMBB;
12256 BTB->Default = Fallthrough;
12257
12258 BTB->DefaultProb = UnhandledProbs;
12259 // If the cases in bit test don't form a contiguous range, we evenly
12260 // distribute the probability on the edge to Fallthrough to two
12261 // successors of CurMBB.
12262 if (!BTB->ContiguousRange) {
12263 BTB->Prob += DefaultProb / 2;
12264 BTB->DefaultProb -= DefaultProb / 2;
12265 }
12266
12267 if (FallthroughUnreachable)
12268 BTB->FallthroughUnreachable = true;
12269
12270 // If we're in the right place, emit the bit test header right now.
12271 if (CurMBB == SwitchMBB) {
12272 visitBitTestHeader(*BTB, SwitchMBB);
12273 BTB->Emitted = true;
12274 }
12275 break;
12276 }
12277 case CC_Range: {
12278 const Value *RHS, *LHS, *MHS;
12280 if (I->Low == I->High) {
12281 // Check Cond == I->Low.
12282 CC = ISD::SETEQ;
12283 LHS = Cond;
12284 RHS=I->Low;
12285 MHS = nullptr;
12286 } else {
12287 // Check I->Low <= Cond <= I->High.
12288 CC = ISD::SETLE;
12289 LHS = I->Low;
12290 MHS = Cond;
12291 RHS = I->High;
12292 }
12293
12294 // If Fallthrough is unreachable, fold away the comparison.
12295 if (FallthroughUnreachable)
12296 CC = ISD::SETTRUE;
12297
12298 // The false probability is the sum of all unhandled cases.
12299 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
12300 getCurSDLoc(), I->Prob, UnhandledProbs);
12301
12302 if (CurMBB == SwitchMBB)
12303 visitSwitchCase(CB, SwitchMBB);
12304 else
12305 SL->SwitchCases.push_back(CB);
12306
12307 break;
12308 }
12309 }
12310 CurMBB = Fallthrough;
12311 }
12312}
12313
12314void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
12315 const SwitchWorkListItem &W,
12316 Value *Cond,
12317 MachineBasicBlock *SwitchMBB) {
12318 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&
12319 "Clusters not sorted?");
12320 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");
12321
12322 auto [LastLeft, FirstRight, LeftProb, RightProb] =
12323 SL->computeSplitWorkItemInfo(W);
12324
12325 // Use the first element on the right as pivot since we will make less-than
12326 // comparisons against it.
12327 CaseClusterIt PivotCluster = FirstRight;
12328 assert(PivotCluster > W.FirstCluster);
12329 assert(PivotCluster <= W.LastCluster);
12330
12331 CaseClusterIt FirstLeft = W.FirstCluster;
12332 CaseClusterIt LastRight = W.LastCluster;
12333
12334 const ConstantInt *Pivot = PivotCluster->Low;
12335
12336 // New blocks will be inserted immediately after the current one.
12338 ++BBI;
12339
12340 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
12341 // we can branch to its destination directly if it's squeezed exactly in
12342 // between the known lower bound and Pivot - 1.
12343 MachineBasicBlock *LeftMBB;
12344 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
12345 FirstLeft->Low == W.GE &&
12346 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
12347 LeftMBB = FirstLeft->MBB;
12348 } else {
12349 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12350 FuncInfo.MF->insert(BBI, LeftMBB);
12351 WorkList.push_back(
12352 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
12353 // Put Cond in a virtual register to make it available from the new blocks.
12355 }
12356
12357 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
12358 // single cluster, RHS.Low == Pivot, and we can branch to its destination
12359 // directly if RHS.High equals the current upper bound.
12360 MachineBasicBlock *RightMBB;
12361 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
12362 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
12363 RightMBB = FirstRight->MBB;
12364 } else {
12365 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
12366 FuncInfo.MF->insert(BBI, RightMBB);
12367 WorkList.push_back(
12368 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
12369 // Put Cond in a virtual register to make it available from the new blocks.
12371 }
12372
12373 // Create the CaseBlock record that will be used to lower the branch.
12374 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
12375 getCurSDLoc(), LeftProb, RightProb);
12376
12377 if (W.MBB == SwitchMBB)
12378 visitSwitchCase(CB, SwitchMBB);
12379 else
12380 SL->SwitchCases.push_back(CB);
12381}
12382
12383// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
12384// from the swith statement.
12386 BranchProbability PeeledCaseProb) {
12387 if (PeeledCaseProb == BranchProbability::getOne())
12389 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
12390
12391 uint32_t Numerator = CaseProb.getNumerator();
12392 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
12393 return BranchProbability(Numerator, std::max(Numerator, Denominator));
12394}
12395
12396// Try to peel the top probability case if it exceeds the threshold.
12397// Return current MachineBasicBlock for the switch statement if the peeling
12398// does not occur.
12399// If the peeling is performed, return the newly created MachineBasicBlock
12400// for the peeled switch statement. Also update Clusters to remove the peeled
12401// case. PeeledCaseProb is the BranchProbability for the peeled case.
12402MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
12403 const SwitchInst &SI, CaseClusterVector &Clusters,
12404 BranchProbability &PeeledCaseProb) {
12405 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12406 // Don't perform if there is only one cluster or optimizing for size.
12407 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
12409 SwitchMBB->getParent()->getFunction().hasMinSize())
12410 return SwitchMBB;
12411
12413 unsigned PeeledCaseIndex = 0;
12414 bool SwitchPeeled = false;
12415 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
12416 CaseCluster &CC = Clusters[Index];
12417 if (CC.Prob < TopCaseProb)
12418 continue;
12419 TopCaseProb = CC.Prob;
12420 PeeledCaseIndex = Index;
12421 SwitchPeeled = true;
12422 }
12423 if (!SwitchPeeled)
12424 return SwitchMBB;
12425
12426 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "
12427 << TopCaseProb << "\n");
12428
12429 // Record the MBB for the peeled switch statement.
12430 MachineFunction::iterator BBI(SwitchMBB);
12431 ++BBI;
12432 MachineBasicBlock *PeeledSwitchMBB =
12434 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
12435
12436 ExportFromCurrentBlock(SI.getCondition());
12437 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
12438 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
12439 nullptr, nullptr, TopCaseProb.getCompl()};
12440 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
12441
12442 Clusters.erase(PeeledCaseIt);
12443 for (CaseCluster &CC : Clusters) {
12444 LLVM_DEBUG(
12445 dbgs() << "Scale the probablity for one cluster, before scaling: "
12446 << CC.Prob << "\n");
12447 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
12448 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n");
12449 }
12450 PeeledCaseProb = TopCaseProb;
12451 return PeeledSwitchMBB;
12452}
12453
12454void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
12455 // Extract cases from the switch.
12457 CaseClusterVector Clusters;
12458 Clusters.reserve(SI.getNumCases());
12459 for (auto I : SI.cases()) {
12460 MachineBasicBlock *Succ = FuncInfo.getMBB(I.getCaseSuccessor());
12461 const ConstantInt *CaseVal = I.getCaseValue();
12462 BranchProbability Prob =
12463 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
12464 : BranchProbability(1, SI.getNumCases() + 1);
12465 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
12466 }
12467
12468 MachineBasicBlock *DefaultMBB = FuncInfo.getMBB(SI.getDefaultDest());
12469
12470 // Cluster adjacent cases with the same destination. We do this at all
12471 // optimization levels because it's cheap to do and will make codegen faster
12472 // if there are many clusters.
12473 sortAndRangeify(Clusters);
12474
12475 // The branch probablity of the peeled case.
12477 MachineBasicBlock *PeeledSwitchMBB =
12478 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
12479
12480 // If there is only the default destination, jump there directly.
12481 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
12482 if (Clusters.empty()) {
12483 assert(PeeledSwitchMBB == SwitchMBB);
12484 SwitchMBB->addSuccessor(DefaultMBB);
12485 if (DefaultMBB != NextBlock(SwitchMBB)) {
12486 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
12487 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
12488 }
12489 return;
12490 }
12491
12492 SL->findJumpTables(Clusters, &SI, getCurSDLoc(), DefaultMBB, DAG.getPSI(),
12493 DAG.getBFI());
12494 SL->findBitTestClusters(Clusters, &SI);
12495
12496 LLVM_DEBUG({
12497 dbgs() << "Case clusters: ";
12498 for (const CaseCluster &C : Clusters) {
12499 if (C.Kind == CC_JumpTable)
12500 dbgs() << "JT:";
12501 if (C.Kind == CC_BitTests)
12502 dbgs() << "BT:";
12503
12504 C.Low->getValue().print(dbgs(), true);
12505 if (C.Low != C.High) {
12506 dbgs() << '-';
12507 C.High->getValue().print(dbgs(), true);
12508 }
12509 dbgs() << ' ';
12510 }
12511 dbgs() << '\n';
12512 });
12513
12514 assert(!Clusters.empty());
12515 SwitchWorkList WorkList;
12516 CaseClusterIt First = Clusters.begin();
12517 CaseClusterIt Last = Clusters.end() - 1;
12518 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
12519 // Scale the branchprobability for DefaultMBB if the peel occurs and
12520 // DefaultMBB is not replaced.
12521 if (PeeledCaseProb != BranchProbability::getZero() &&
12522 DefaultMBB == FuncInfo.getMBB(SI.getDefaultDest()))
12523 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
12524 WorkList.push_back(
12525 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
12526
12527 while (!WorkList.empty()) {
12528 SwitchWorkListItem W = WorkList.pop_back_val();
12529 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
12530
12531 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOptLevel::None &&
12532 !DefaultMBB->getParent()->getFunction().hasMinSize()) {
12533 // For optimized builds, lower large range as a balanced binary tree.
12534 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
12535 continue;
12536 }
12537
12538 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
12539 }
12540}
12541
12542void SelectionDAGBuilder::visitStepVector(const CallInst &I) {
12544 auto DL = getCurSDLoc();
12545 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12546 setValue(&I, DAG.getStepVector(DL, ResultVT));
12547}
12548
12549void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
12551 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12552
12553 SDLoc DL = getCurSDLoc();
12554 SDValue V = getValue(I.getOperand(0));
12555 assert(VT == V.getValueType() && "Malformed vector.reverse!");
12556
12557 if (VT.isScalableVector()) {
12559 return;
12560 }
12561
12562 // Use VECTOR_SHUFFLE for the fixed-length vector
12563 // to maintain existing behavior.
12565 unsigned NumElts = VT.getVectorMinNumElements();
12566 for (unsigned i = 0; i != NumElts; ++i)
12567 Mask.push_back(NumElts - 1 - i);
12568
12569 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask));
12570}
12571
12572void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) {
12573 auto DL = getCurSDLoc();
12574 SDValue InVec = getValue(I.getOperand(0));
12575 EVT OutVT =
12577
12578 unsigned OutNumElts = OutVT.getVectorMinNumElements();
12579
12580 // ISD Node needs the input vectors split into two equal parts
12581 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12583 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12584 DAG.getVectorIdxConstant(OutNumElts, DL));
12585
12586 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12587 // legalisation and combines.
12588 if (OutVT.isFixedLengthVector()) {
12589 SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12590 createStrideMask(0, 2, OutNumElts));
12591 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi,
12592 createStrideMask(1, 2, OutNumElts));
12593 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc());
12594 setValue(&I, Res);
12595 return;
12596 }
12597
12599 DAG.getVTList(OutVT, OutVT), Lo, Hi);
12600 setValue(&I, Res);
12601}
12602
12603void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) {
12604 auto DL = getCurSDLoc();
12605 EVT InVT = getValue(I.getOperand(0)).getValueType();
12606 SDValue InVec0 = getValue(I.getOperand(0));
12607 SDValue InVec1 = getValue(I.getOperand(1));
12609 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12610
12611 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12612 // legalisation and combines.
12613 if (OutVT.isFixedLengthVector()) {
12614 unsigned NumElts = InVT.getVectorMinNumElements();
12615 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1);
12616 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT),
12617 createInterleaveMask(NumElts, 2)));
12618 return;
12619 }
12620
12622 DAG.getVTList(InVT, InVT), InVec0, InVec1);
12623 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0),
12624 Res.getValue(1));
12625 setValue(&I, Res);
12626}
12627
12628void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) {
12629 SmallVector<EVT, 4> ValueVTs;
12631 ValueVTs);
12632 unsigned NumValues = ValueVTs.size();
12633 if (NumValues == 0) return;
12634
12635 SmallVector<SDValue, 4> Values(NumValues);
12636 SDValue Op = getValue(I.getOperand(0));
12637
12638 for (unsigned i = 0; i != NumValues; ++i)
12639 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i],
12640 SDValue(Op.getNode(), Op.getResNo() + i));
12641
12643 DAG.getVTList(ValueVTs), Values));
12644}
12645
12646void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
12648 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
12649
12650 SDLoc DL = getCurSDLoc();
12651 SDValue V1 = getValue(I.getOperand(0));
12652 SDValue V2 = getValue(I.getOperand(1));
12653 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue();
12654
12655 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
12656 if (VT.isScalableVector()) {
12657 setValue(
12658 &I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
12660 Imm, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
12661 return;
12662 }
12663
12664 unsigned NumElts = VT.getVectorNumElements();
12665
12666 uint64_t Idx = (NumElts + Imm) % NumElts;
12667
12668 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors.
12670 for (unsigned i = 0; i < NumElts; ++i)
12671 Mask.push_back(Idx + i);
12672 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask));
12673}
12674
12675// Consider the following MIR after SelectionDAG, which produces output in
12676// phyregs in the first case or virtregs in the second case.
12677//
12678// INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx
12679// %5:gr32 = COPY $ebx
12680// %6:gr32 = COPY $edx
12681// %1:gr32 = COPY %6:gr32
12682// %0:gr32 = COPY %5:gr32
12683//
12684// INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32
12685// %1:gr32 = COPY %6:gr32
12686// %0:gr32 = COPY %5:gr32
12687//
12688// Given %0, we'd like to return $ebx in the first case and %5 in the second.
12689// Given %1, we'd like to return $edx in the first case and %6 in the second.
12690//
12691// If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap
12692// to a single virtreg (such as %0). The remaining outputs monotonically
12693// increase in virtreg number from there. If a callbr has no outputs, then it
12694// should not have a corresponding callbr landingpad; in fact, the callbr
12695// landingpad would not even be able to refer to such a callbr.
12697 MachineInstr *MI = MRI.def_begin(Reg)->getParent();
12698 // There is definitely at least one copy.
12699 assert(MI->getOpcode() == TargetOpcode::COPY &&
12700 "start of copy chain MUST be COPY");
12701 Reg = MI->getOperand(1).getReg();
12702 MI = MRI.def_begin(Reg)->getParent();
12703 // There may be an optional second copy.
12704 if (MI->getOpcode() == TargetOpcode::COPY) {
12705 assert(Reg.isVirtual() && "expected COPY of virtual register");
12706 Reg = MI->getOperand(1).getReg();
12707 assert(Reg.isPhysical() && "expected COPY of physical register");
12708 MI = MRI.def_begin(Reg)->getParent();
12709 }
12710 // The start of the chain must be an INLINEASM_BR.
12711 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
12712 "end of copy chain MUST be INLINEASM_BR");
12713 return Reg;
12714}
12715
12716// We must do this walk rather than the simpler
12717// setValue(&I, getCopyFromRegs(CBR, CBR->getType()));
12718// otherwise we will end up with copies of virtregs only valid along direct
12719// edges.
12720void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) {
12721 SmallVector<EVT, 8> ResultVTs;
12722 SmallVector<SDValue, 8> ResultValues;
12723 const auto *CBR =
12724 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator());
12725
12729
12730 unsigned InitialDef = FuncInfo.ValueMap[CBR];
12731 SDValue Chain = DAG.getRoot();
12732
12733 // Re-parse the asm constraints string.
12734 TargetLowering::AsmOperandInfoVector TargetConstraints =
12735 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR);
12736 for (auto &T : TargetConstraints) {
12737 SDISelAsmOperandInfo OpInfo(T);
12738 if (OpInfo.Type != InlineAsm::isOutput)
12739 continue;
12740
12741 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the
12742 // individual constraint.
12743 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
12744
12745 switch (OpInfo.ConstraintType) {
12748 // Fill in OpInfo.AssignedRegs.Regs.
12749 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo);
12750
12751 // getRegistersForValue may produce 1 to many registers based on whether
12752 // the OpInfo.ConstraintVT is legal on the target or not.
12753 for (Register &Reg : OpInfo.AssignedRegs.Regs) {
12754 Register OriginalDef = FollowCopyChain(MRI, InitialDef++);
12755 if (OriginalDef.isPhysical())
12756 FuncInfo.MBB->addLiveIn(OriginalDef);
12757 // Update the assigned registers to use the original defs.
12758 Reg = OriginalDef;
12759 }
12760
12761 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs(
12762 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR);
12763 ResultValues.push_back(V);
12764 ResultVTs.push_back(OpInfo.ConstraintVT);
12765 break;
12766 }
12768 SDValue Flag;
12769 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(),
12770 OpInfo, DAG);
12771 ++InitialDef;
12772 ResultValues.push_back(V);
12773 ResultVTs.push_back(OpInfo.ConstraintVT);
12774 break;
12775 }
12776 default:
12777 break;
12778 }
12779 }
12781 DAG.getVTList(ResultVTs), ResultValues);
12782 setValue(&I, V);
12783}
unsigned const MachineRegisterInfo * MRI
@ Poison
static unsigned getIntrinsicID(const SDNode *N)
unsigned RegSize
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(...)
Definition: Debug.h:106
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI)
Returns an AttributeList representing the attributes applied to the return value of the given call.
Definition: FastISel.cpp:943
#define Check(C,...)
Hexagon Common GEP
const HexagonInstrInfo * TII
static bool isUndef(ArrayRef< int > Mask)
IRTranslator LLVM IR MI
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Module.h This file contains the declarations for the Module class.
static void getRegistersForValue(MachineFunction &MF, MachineIRBuilder &MIRBuilder, GISelAsmOperandInfo &OpInfo, GISelAsmOperandInfo &RefOpInfo)
Assign virtual/physical registers for the specified register operand.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::optional< ConstantRange > getRange(Value *V, const InstrInfoQuery &IIQ)
Helper method to get range from metadata or attribute.
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static const Function * getCalledFunction(const Value *V)
This file provides utility analysis objects describing memory locations.
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
This file contains the declarations for metadata subclasses.
#define T1
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
This file contains some templates that are useful if you are working with the STL at all.
static bool hasOnlySelectUsers(const Value *Cond)
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain)
Create a LOAD_STACK_GUARD node, and let it carry the target specific global variable if there exists ...
static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, const SDLoc &DL, SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder)
Add a stack map intrinsic call's live variable operands to a stackmap or patchpoint target node's ope...
static const unsigned MaxParallelChains
static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
visitPow - Lower a pow intrinsic.
static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, ISD::MemIndexType &IndexType, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize)
static const CallBase * FindPreallocatedCall(const Value *PreallocatedSetup)
Given a @llvm.call.preallocated.setup, return the corresponding preallocated call.
static cl::opt< unsigned > SwitchPeelThreshold("switch-peel-threshold", cl::Hidden, cl::init(66), cl::desc("Set the case probability threshold for peeling the case from a " "switch statement. A value greater than 100 will void this " "optimization"))
static cl::opt< bool > InsertAssertAlign("insert-assert-align", cl::init(true), cl::desc("Insert the experimental `assertalign` node."), cl::ReallyHidden)
static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin)
static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, DILocalVariable *Variable, DebugLoc DL, unsigned Order, SmallVectorImpl< Value * > &Values, DIExpression *Expression)
static unsigned findMatchingInlineAsmOperand(unsigned OperandNo, const std::vector< SDValue > &AsmNodeOperands)
static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, SDISelAsmOperandInfo &MatchingOpInfo, SelectionDAG &DAG)
Make sure that the output operand OpInfo and its corresponding input operand MatchingOpInfo have comp...
static void findUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
When an invoke or a cleanupret unwinds to the next EH pad, there are many places it could ultimately ...
static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic)
static BranchProbability scaleCaseProbality(BranchProbability CaseProb, BranchProbability PeeledCaseProb)
static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp2 - Lower an exp2 intrinsic.
static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue Scale, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, const SDLoc &dl)
getF32Constant - Get 32-bit floating point constant.
static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, const SDLoc &DL, EVT PartVT)
static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog10 - Lower a log10 intrinsic.
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv)
getCopyToPartsVector - Create a series of nodes that contain the specified value split into legal par...
static void getUnderlyingArgRegs(SmallVectorImpl< std::pair< Register, TypeSize > > &Regs, const SDValue &N)
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, const Value *V, std::optional< CallingConv::ID > CallConv=std::nullopt, ISD::NodeType ExtendKind=ISD::ANY_EXTEND)
getCopyToParts - Create a series of nodes that contain the specified value split into legal parts.
static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, SelectionDAGBuilder &Builder)
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog2 - Lower a log2 intrinsic.
static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, SDISelAsmOperandInfo &OpInfo, SelectionDAG &DAG)
Get a direct memory input to behave well as an indirect operand.
static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel)
isOnlyUsedInEntryBlock - If the specified argument is only used in the entry block,...
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, const Twine &ErrMsg)
static bool collectInstructionDeps(SmallMapVector< const Instruction *, bool, 8 > *Deps, const Value *V, SmallMapVector< const Instruction *, bool, 8 > *Necessary=nullptr, unsigned Depth=0)
static void findArgumentCopyElisionCandidates(const DataLayout &DL, FunctionLoweringInfo *FuncInfo, ArgCopyElisionMapTy &ArgCopyElisionCandidates)
Scan the entry block of the function in FuncInfo for arguments that look like copies into a local all...
static bool isFunction(SDValue Op)
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, const TargetLowering &TLI, const SDLoc &dl)
GetExponent - Get the exponent:
static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg)
static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, SelectionDAG &DAG)
ExpandPowI - Expand a llvm.powi intrinsic.
static void findWasmUnwindDestinations(FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, BranchProbability Prob, SmallVectorImpl< std::pair< MachineBasicBlock *, BranchProbability > > &UnwindDests)
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandLog - Lower a log intrinsic.
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC=std::nullopt, std::optional< ISD::NodeType > AssertOp=std::nullopt)
getCopyFromParts - Create a value that contains the specified legal parts combined into the value the...
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, SelectionDAG &DAG)
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl)
GetSignificand - Get the significand and build it into a floating-point number with exponent of 1:
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags)
expandExp - Lower an exp intrinsic.
static const MDNode * getRangeMetadata(const Instruction &I)
static cl::opt< unsigned, true > LimitFPPrecision("limit-float-precision", cl::desc("Generate low-precision inline sequences " "for some float libcalls"), cl::location(LimitFloatPrecision), cl::Hidden, cl::init(0))
static void tryToElideArgumentCopy(FunctionLoweringInfo &FuncInfo, SmallVectorImpl< SDValue > &Chains, DenseMap< int, int > &ArgCopyElisionFrameIndexMap, SmallPtrSetImpl< const Instruction * > &ElidedArgCopyInstrs, ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, ArrayRef< SDValue > ArgVals, bool &ArgHasUses)
Try to elide argument copies from memory into a local alloca.
static unsigned LimitFloatPrecision
LimitFloatPrecision - Generate low-precision inline sequences for some float libcalls (6,...
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, SDValue InChain, std::optional< CallingConv::ID > CC)
getCopyFromPartsVector - Create a value that contains the specified legal parts combined into the val...
static bool InBlock(const Value *V, const BasicBlock *BB)
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const unsigned char *MatcherTable, unsigned &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:39
This pass exposes codegen information to IR-level passes.
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:204
support::ulittle16_t & Hi
Definition: aarch32.cpp:203
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
Class for arbitrary precision integers.
Definition: APInt.h:78
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1015
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:334
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:440
an instruction to allocate memory on the stack
Definition: Instructions.h:63
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
Definition: Instructions.h:124
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
Definition: Instructions.h:117
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
bool hasAttribute(Attribute::AttrKind Kind) const
Check if an argument has a given attribute.
Definition: Function.cpp:349
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition: Argument.h:49
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:157
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:168
iterator begin() const
Definition: ArrayRef.h:156
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:163
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:501
This class represents the atomic memcpy intrinsic i.e.
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:704
@ Add
*p = old + v
Definition: Instructions.h:720
@ FAdd
*p = old + v
Definition: Instructions.h:741
@ USubCond
Subtract only if no unsigned overflow.
Definition: Instructions.h:764
@ Min
*p = old <signed v ? old : v
Definition: Instructions.h:734
@ Or
*p = old | v
Definition: Instructions.h:728
@ Sub
*p = old - v
Definition: Instructions.h:722
@ And
*p = old & v
Definition: Instructions.h:724
@ Xor
*p = old ^ v
Definition: Instructions.h:730
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
Definition: Instructions.h:768
@ FSub
*p = old - v
Definition: Instructions.h:744
@ UIncWrap
Increment one up to a maximum value.
Definition: Instructions.h:756
@ Max
*p = old >signed v ? old : v
Definition: Instructions.h:732
@ UMin
*p = old <unsigned v ? old : v
Definition: Instructions.h:738
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
Definition: Instructions.h:752
@ UMax
*p = old >unsigned v ? old : v
Definition: Instructions.h:736
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
Definition: Instructions.h:748
@ UDecWrap
Decrement one until a minimum value or zero.
Definition: Instructions.h:760
@ Nand
*p = ~(old & v)
Definition: Instructions.h:726
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:367
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
Definition: BasicBlock.cpp:571
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:219
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:386
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:239
const Instruction & back() const
Definition: BasicBlock.h:473
This class represents a no-op cast from one type to another.
bool test(unsigned Idx) const
Definition: BitVector.h:461
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
BitVector & set()
Definition: BitVector.h:351
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
The address of a basic block.
Definition: Constants.h:893
Conditional or Unconditional Branch instruction.
Analysis providing branch probability information.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
static uint32_t getDenominator()
static BranchProbability getOne()
static BranchProbability getUnknown()
uint32_t getNumerator() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
BranchProbability getCompl() const
static BranchProbability getZero()
static void normalizeProbabilities(ProbabilityIter Begin, ProbabilityIter End)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1112
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:2053
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1399
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1261
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
unsigned countOperandBundlesOfType(StringRef Name) const
Return the number of operand bundles with the tag Name attached to this instruction.
Definition: InstrTypes.h:2029
Value * getCalledOperand() const
Definition: InstrTypes.h:1334
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1286
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1267
bool isConvergent() const
Determine if the invoke is convergent.
Definition: InstrTypes.h:1937
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1199
unsigned arg_size() const
Definition: InstrTypes.h:1284
AttributeList getAttributes() const
Return the attributes for this call.
Definition: InstrTypes.h:1417
bool isTailCall() const
Tests if this call site is marked as a tail call.
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
This class represents a function call, abstracting a target machine's calling convention.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:661
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:673
ConstantDataSequential - A vector or array constant whose element type is a simple 1/2/4/8-byte integ...
Definition: Constants.h:587
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1108
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2321
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:271
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:866
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:208
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:873
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:157
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:148
A signed pointer, in the ptrauth sense.
Definition: Constants.h:1021
This class represents a range of values.
Definition: ConstantRange.h:47
uint64_t getZExtValue() const
Constant Vector Declarations.
Definition: Constants.h:511
This is an important base class in LLVM.
Definition: Constant.h:42
This is the common base class for constrained floating point intrinsics.
std::optional< fp::ExceptionBehavior > getExceptionBehavior() const
unsigned getNonMetadataArgCount() const
DWARF expression.
bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B)
Check if fragments overlap between a pair of FragmentInfos.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static std::optional< FragmentInfo > getFragmentInfo(expr_op_iterator Start, expr_op_iterator End)
Retrieve the details of this fragment expression.
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static const DIExpression * convertToUndefExpression(const DIExpression *Expr)
Removes all elements from Expr that do not apply to an undef debug value, which includes every operat...
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
static DIExpression * prependOpcodes(const DIExpression *Expr, SmallVectorImpl< uint64_t > &Ops, bool StackValue=false, bool EntryValue=false)
Prepend DIExpr with the given opcodes and optionally turn it into a stack value.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Debug location.
Base class for variables.
std::optional< uint64_t > getSizeInBits() const
Determines the size of the variable's type.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Definition: DataLayout.cpp:709
bool isBigEndian() const
Definition: DataLayout.h:198
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
Definition: DataLayout.h:369
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Definition: DataLayout.h:421
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:847
This represents the llvm.dbg.label instruction.
DILabel * getLabel() const
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
iterator_range< location_op_iterator > getValues() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
Value * getVariableLocationOp(unsigned OpIdx) const
DILocalVariable * getVariable() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
A debug info location.
Definition: DebugLoc.h:33
DILocation * getInlinedAt() const
Definition: DebugLoc.cpp:39
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:156
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:152
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:211
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
Diagnostic information for inline asm reporting.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:311
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
Definition: TypeSize.h:317
constexpr bool isScalar() const
Exactly one element.
Definition: TypeSize.h:322
Class representing an expression and its matching format.
This instruction extracts a struct member or array element value from an aggregate value.
This instruction compares its operands according to the predicate given to the constructor.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
bool allowReassoc() const
Flag queries.
Definition: FMF.h:65
An instruction for ordering other memory operations.
Definition: Instructions.h:424
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:791
This class represents a freeze function that returns random concrete value if an operand is either a ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
BranchProbabilityInfo * BPI
Register CreateRegs(const Value *V)
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
MachineBasicBlock * getMBB(const BasicBlock *BB) const
Register DemoteRegister
DemoteRegister - if CanLowerReturn is false, DemoteRegister is a vreg allocated to hold a pointer to ...
BitVector DescribedArgs
Bitvector with a bit set if corresponding argument is described in ArgDbgValues.
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
const LiveOutInfo * GetLiveOutRegInfo(Register Reg)
GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the register is a PHI destinat...
Register InitializeRegForValue(const Value *V)
unsigned ExceptionPointerVirtReg
If the current MBB is a landing pad, the exception pointer and exception selector registers are copie...
SmallPtrSet< const DbgDeclareInst *, 8 > PreprocessedDbgDeclares
Collection of dbg.declare instructions handled after argument lowering and before ISel proper.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineBasicBlock::iterator InsertPt
MBB - The current insert position inside the current block.
MachineBasicBlock * MBB
MBB - The current block.
std::vector< std::pair< MachineInstr *, unsigned > > PHINodesToUpdate
PHINodesToUpdate - A list of phi instructions whose operand list will be updated after processing the...
SmallVector< MachineInstr *, 8 > ArgDbgValues
ArgDbgValues - A list of DBG_VALUE instructions created during isel for function arguments that are i...
unsigned getCurrentCallSite()
Get the call site currently being processed, if any. Return zero if none.
void setCurrentCallSite(unsigned Site)
Set the call site currently being processed.
MachineRegisterInfo * RegInfo
Register CreateReg(MVT VT, bool isDivergent=false)
CreateReg - Allocate a single virtual register for the given type.
bool CanLowerReturn
CanLowerReturn - true iff the function's return value can be lowered to registers.
DenseMap< const Value *, ISD::NodeType > PreferredExtendType
Record the preferred extend type (ISD::SIGN_EXTEND or ISD::ZERO_EXTEND) for a value.
Register getCatchPadExceptionPointerVReg(const Value *CPI, const TargetRegisterClass *RC)
Class to represent function types.
Definition: DerivedTypes.h:105
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:144
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:137
Type * getReturnType() const
Definition: DerivedTypes.h:126
Data structure describing the variable locations in a function.
const BasicBlock & getEntryBlock() const
Definition: Function.h:809
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:251
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:704
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:345
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:277
Constant * getPersonalityFn() const
Get the personality function associated with this function.
Definition: Function.cpp:1048
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:353
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Definition: Function.h:256
size_t arg_size() const
Definition: Function.h:901
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
Definition: Function.h:234
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:731
Garbage collection metadata for a single function.
Definition: GCMetadata.h:78
void addStackRoot(int Num, const Constant *Metadata)
addStackRoot - Registers a root that lives on the stack.
Definition: GCMetadata.h:118
Represents flags for the getelementptr instruction/expression.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:933
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:568
bool hasDLLImportStorageClass() const
Definition: GlobalValue.h:279
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:657
This instruction compares its operands according to the predicate given to the constructor.
Indirect Branch Instruction.
This instruction inserts a struct field of array element value into an aggregate value.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:475
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
Definition: Metadata.cpp:1750
@ MIN_INT_BITS
Minimum number of bits that can be specified.
Definition: DerivedTypes.h:53
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:55
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:176
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:345
MCSymbol * getOrCreateFrameAllocSymbol(const Twine &FuncName, unsigned Idx)
Gets a symbol that will be defined to the final stack offset of a local variable after codegen.
Definition: MCContext.cpp:235
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
Metadata node.
Definition: Metadata.h:1073
Machine Value Type.
@ INVALID_SIMPLE_VALUE_TYPE
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
bool isEHPad() const
Returns true if the block is a landing pad.
void setIsEHCatchretTarget(bool V=true)
Indicates if this is a target block of a catchret.
void setIsCleanupFuncletEntry(bool V=true)
Indicates if this is the entry block of a cleanup funclet.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void setIsEHFuncletEntry(bool V=true)
Indicates if this is the entry block of an EH funclet.
void setIsEHScopeEntry(bool V=true)
Indicates if this is the entry block of an EH scope, i.e., the block that that used to have a catchpa...
void setMachineBlockAddressTaken()
Set this block to indicate that its address is used as something other than the target of a terminato...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
void setIsImmutableObjectIndex(int ObjectIdx, bool IsImmutable)
Marks the immutability of an object.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setHasPatchPoint(bool s=true)
void setHasStackMap(bool s=true)
bool hasOpaqueSPAdjustment() const
Returns true if the function contains opaque dynamic stack adjustments.
int getStackProtectorIndex() const
Return the index for the stack protector object.
void setStackProtectorIndex(int I)
void setIsAliasedObjectIndex(int ObjectIdx, bool IsAliased)
Set "maybe pointed to by an LLVM IR value" for an object.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
void RemoveStackObject(int ObjectIdx)
Remove or mark dead a statically sized stack object.
void setFunctionContextIndex(int I)
Description of the location of a variable whose Address is valid and unchanging during function execu...
const WinEHFuncInfo * getWinEHFuncInfo() const
getWinEHFuncInfo - Return information about how the current function uses Windows exception handling.
void setCallsUnwindInit(bool b)
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setCallSiteBeginLabel(MCSymbol *BeginLabel, unsigned Site)
Map the begin label for a call site.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void setHasEHCatchret(bool V)
void setCallsEHReturn(bool b)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
unsigned getTypeIDFor(const GlobalValue *TI)
Return the type id for the specified typeinfo. This is function wide.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
auto getInStackSlotVariableDbgInfo()
Returns the collection of variables for which we have debug info and that have been assigned a stack ...
void addCodeViewAnnotation(MCSymbol *Label, MDNode *MD)
Record annotations associated with a particular label.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
void addInvoke(MachineBasicBlock *LandingPad, MCSymbol *BeginLabel, MCSymbol *EndLabel)
Provide the begin and end labels of an invoke style call and associate it with a try landing pad bloc...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void erase(iterator MBBI)
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
static MachineOperand CreateFI(int Idx)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
ArrayRef< std::pair< MCRegister, Register > > liveins() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
bool contains(const KeyT &Key) const
Definition: MapVector.h:163
std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)
Definition: MapVector.h:118
Representation for a specific memory location.
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
Metadata wrapper in the Value hierarchy.
Definition: Metadata.h:180
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Utility class for integer operators which may exhibit overflow - Add, Sub, Mul, and Shl.
Definition: Operator.h:77
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:686
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1878
A udiv or sdiv instruction, which can be marked as "exact", indicating that no bits are destroyed.
Definition: Operator.h:155
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
static constexpr bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:71
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:65
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
Resume the propagation of an exception.
Return a value (possibly void), from a function.
Holds the information from a dbg_label node through SDISel.
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
void dump() const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
void addDanglingDebugInfo(SmallVectorImpl< Value * > &Values, DILocalVariable *Var, DIExpression *Expr, bool IsVariadic, DebugLoc DL, unsigned Order)
Register a dbg_value which relies on a Value which we have not yet seen.
void visitDbgInfo(const Instruction &I)
void clearDanglingDebugInfo()
Clear the dangling debug information map.
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall, bool IsMustTailCall, const BasicBlock *EHPadBB=nullptr, const TargetLowering::PtrAuthInfo *PAI=nullptr)
void clear()
Clear out the current SelectionDAG and the associated state and prepare this SelectionDAGBuilder obje...
void visitBitTestHeader(SwitchCG::BitTestBlock &B, MachineBasicBlock *SwitchBB)
visitBitTestHeader - This function emits necessary code to produce value suitable for "bit tests"
void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)
std::unique_ptr< SDAGSwitchLowering > SL
SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)
bool HasTailCall
This is set to true if a call in the current block has been translated as a tail call.
bool ShouldEmitAsBranches(const std::vector< SwitchCG::CaseBlock > &Cases)
If the set of cases should be emitted as a series of branches, return true.
void EmitBranchForMergedCondition(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
EmitBranchForMergedCondition - Helper method for FindMergedConditions.
void LowerDeoptimizeCall(const CallInst *CI)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
SwiftErrorValueTracking & SwiftError
Information about the swifterror values used throughout the function.
SDValue getNonRegisterValue(const Value *V)
getNonRegisterValue - Return an SDValue for the given Value, but don't look in FuncInfo....
void CopyValueToVirtualRegister(const Value *V, unsigned Reg, ISD::NodeType ExtendType=ISD::ANY_EXTEND)
DenseMap< MachineBasicBlock *, SmallVector< unsigned, 4 > > LPadToCallSiteMap
Map a landing pad to the call site indexes.
void handleDebugDeclare(Value *Address, DILocalVariable *Variable, DIExpression *Expression, DebugLoc DL)
bool shouldKeepJumpConditionsTogether(const FunctionLoweringInfo &FuncInfo, const BranchInst &I, Instruction::BinaryOps Opc, const Value *Lhs, const Value *Rhs, TargetLoweringBase::CondMergingParams Params) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
void init(GCFunctionInfo *gfi, AAResults *AA, AssumptionCache *AC, const TargetLibraryInfo *li)
void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, BranchProbability BranchProbToNext, Register Reg, SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB)
visitBitTestCase - this function produces one "bit test"
DenseMap< const Constant *, unsigned > ConstantsOut
void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)
Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.
void salvageUnresolvedDbgValue(const Value *V, DanglingDebugInfo &DDI)
For the given dangling debuginfo record, perform last-ditch efforts to resolve the debuginfo to somet...
SmallVector< SDValue, 8 > PendingLoads
Loads are not emitted to the program immediately.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
SDValue getRoot()
Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...
void resolveOrClearDbgInfo()
Evict any dangling debug information, attempting to salvage it first.
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
SDValue getMemoryRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
void resolveDanglingDebugInfo(const Value *V, SDValue Val)
If we saw an earlier dbg_value referring to V, generate the debug data structures now that we've seen...
void visit(const Instruction &I)
void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr)
If we have dangling debug info that describes Variable, or an overlapping part of variable considerin...
SDValue getCopyFromRegs(const Value *V, Type *Ty)
If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.
void CopyToExportRegsIfNeeded(const Value *V)
CopyToExportRegsIfNeeded - If the given value has virtual registers created for it,...
void handleKillDebugValue(DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order)
Create a record for a kill location debug intrinsic.
void visitJumpTable(SwitchCG::JumpTable &JT)
visitJumpTable - Emit JumpTable node in the current MBB
void visitJumpTableHeader(SwitchCG::JumpTable &JT, SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB)
visitJumpTableHeader - This function emits necessary code to produce index in the JumpTable from swit...
void LowerCallSiteWithPtrAuthBundle(const CallBase &CB, const BasicBlock *EHPadBB)
static const unsigned LowestSDNodeOrder
Lowest valid SDNodeOrder.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
void setValue(const Value *V, SDValue NewN)
void FindMergedConditions(const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, Instruction::BinaryOps Opc, BranchProbability TProb, BranchProbability FProb, bool InvertCond)
const TargetLibraryInfo * LibInfo
bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB)
void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB)
Codegen a new tail for a stack protector check ParentMBB which has had its tail spliced into a stack ...
bool handleDebugValue(ArrayRef< const Value * > Values, DILocalVariable *Var, DIExpression *Expr, DebugLoc DbgLoc, unsigned Order, bool IsVariadic)
For a given list of Values, attempt to create and record a SDDbgValue in the SelectionDAG.
SDValue getControlRoot()
Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...
void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last)
When an MBB was split during scheduling, update the references that need to refer to the last resulti...
SDValue getValueImpl(const Value *V)
getValueImpl - Helper function for getValue and getNonRegisterValue.
void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB)
visitSwitchCase - Emits the necessary code to represent a single node in the binary search tree resul...
void visitSPDescriptorFailure(StackProtectorDescriptor &SPD)
Codegen the failure basic block for a stack protector check.
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
const TargetLowering * TLI
MachineFunction * MF
virtual void emitFunctionEntryCode()
SwiftErrorValueTracking * SwiftError
std::unique_ptr< SelectionDAGBuilder > SDB
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrnlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a memcmp/bcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcpy(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, bool isStpcpy) const
Emit target-specific code that performs a strcpy or stpcpy, in cases where that is faster than a libc...
virtual std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo Op1PtrInfo, MachinePointerInfo Op2PtrInfo) const
Emit target-specific code that performs a strcmp, in cases where that is faster than a libcall.
virtual std::pair< SDValue, SDValue > EmitTargetCodeForStrlen(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, MachinePointerInfo SrcPtrInfo) const
virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const
Help to insert SDNodeFlags automatically in transforming.
Definition: SelectionDAG.h:371
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:228
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:750
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
Definition: SelectionDAG.h:982
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:577
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:499
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
Definition: SelectionDAG.h:801
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
BlockFrequencyInfo * getBFI() const
Definition: SelectionDAG.h:513
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
void addMMRAMetadata(const SDNode *Node, MDNode *MMRA)
Set MMRAMetadata to be associated with Node.
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getRegister(Register Reg, EVT VT)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:503
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:458
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:856
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
Definition: SelectionDAG.h:827
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:497
ProfileSummaryInfo * getPSI() const
Definition: SelectionDAG.h:512
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:755
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
const SelectionDAGTargetInfo & getSelectionDAGInfo() const
Definition: SelectionDAG.h:505
SDValue getMaskedHistogram(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:890
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getPartialReduceAdd(SDLoc DL, EVT ReducedTy, SDValue Op1, SDValue Op2)
Create the DAG equivalent of vector_partial_reduce where Op1 and Op2 are its operands and ReducedTY i...
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:498
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT)
Definition: SelectionDAG.h:736
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:700
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:492
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
const FunctionVarLocs * getFunctionVarLocs() const
Returns the result of the AssignmentTrackingAnalysis pass if it's available, otherwise return nullptr...
Definition: SelectionDAG.h:509
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
Definition: SelectionDAG.h:510
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:586
void addPCSections(const SDNode *Node, MDNode *MD)
Set PCSections to be associated with Node.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getSetCCVP(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Mask, SDValue EVL)
Helper function to make it easier to build VP_SETCCs if you just have an ISD::CondCode instead of an ...
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:580
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:906
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:363
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:384
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:519
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:132
bool empty() const
Definition: SmallVector.h:81
size_t size() const
Definition: SmallVector.h:78
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:573
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:937
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:683
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:968
void resize(size_type N)
Definition: SmallVector.h:638
void push_back(const T &Elt)
Definition: SmallVector.h:413
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:286
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
Encapsulates all of the information needed to generate a stack protector check, and signals to isel w...
MachineBasicBlock * getSuccessMBB()
MachineBasicBlock * getFailureMBB()
void clear()
Clear the memory usage of this object.
An instruction for storing to memory.
Definition: Instructions.h:292
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:147
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:144
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:596
Class to represent struct types.
Definition: DerivedTypes.h:218
void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register)
Set the swifterror virtual register in the VRegDefMap for this basic block.
Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a use of a swifterror by an instruction.
Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, const Value *)
Get or create the swifterror value virtual register for a def of a swifterror by an instruction.
const Value * getFunctionArg() const
Get the (unique) function argument that was marked swifterror, or nullptr if this function has no swi...
Multiway switch.
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
TargetInstrInfo - Interface to description of machine instruction set.
TargetIntrinsicInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
void setAttributes(const CallBase *Call, unsigned ArgIdx)
Set CallLoweringInfo attribute flags based on a call instruction and called function attributes.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const
Returns the desired alignment for ByVal or InAlloca aggregate function arguments in the caller parame...
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
EVT getMemValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual bool useStackGuardXorFP() const
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isLegalScaleForGatherScatter(uint64_t Scale, uint64_t ElemSize) const
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
virtual CondMergingParams getJumpConditionMergingParams(Instruction::BinaryOps, const Value *, const Value *) const
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
virtual Value * getSDagStackGuard(const Module &M) const
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
bool isJumpExpensive() const
Return true if Flow Control is an expensive operation that should be avoided.
unsigned getBitWidthForCttzElements(Type *RetTy, ElementCount EC, bool ZeroIsPoison, const ConstantRange *VScaleRange) const
Return the minimum number of bits required to hold the maximum possible number of trailing zero vecto...
virtual bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const
Returns true if the index type for a masked gather/scatter requires extending.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
virtual Function * getSSPStackGuardCheck(const Module &M) const
If the target has a standard stack protection check function that performs validation and error handl...
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI, const DataLayout &DL) const
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned) const
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual Align getABIAlignmentForCallingConv(Type *ArgTy, const DataLayout &DL) const
Certain targets have context sensitive alignment requirements, where one type has the alignment requi...
virtual bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC=nullptr, const TargetLibraryInfo *LibInfo=nullptr) const
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldExpandVectorMatch(EVT VT, unsigned SearchSize) const
Return true if the @llvm.experimental.vector.match intrinsic should be expanded for vector type ‘VT’ ...
virtual MVT getFenceOperandTy(const DataLayout &DL) const
Return the type for operands of fence.
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF, bool IsScalable) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual MVT hasFastEqualityCompare(unsigned NumBits) const
Return the preferred operand type if the target has a quick way to compare integer values of the give...
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI, const DataLayout &DL) const
virtual bool shouldExpandPartialReductionIntrinsic(const IntrinsicInst *I) const
Return true if the @llvm.experimental.vector.partial.reduce.
virtual bool shouldExpandCttzElements(EVT VT) const
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
virtual bool signExtendConstant(const ConstantInt *C) const
Return true if this constant should be sign extended when promoting to a larger type.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual Register getExceptionPointerRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception address on entry to an ...
bool supportsUnalignedAtomics() const
Whether the target supports unaligned atomic operations.
std::vector< ArgListEntry > ArgListTy
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual Register getExceptionSelectorRegister(const Constant *PersonalityFn) const
If a physical register, this returns the register that receives the exception typeid on entry to a la...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
virtual MVT getVPExplicitVectorLengthTy() const
Returns the type to be used for the EVL/AVL operand of VP nodes: ISD::VP_ADD, ISD::VP_SUB,...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool supportKCFIBundles() const
Return true if the target supports kcfi operand bundles.
virtual bool supportPtrAuthBundles() const
Return true if the target supports ptrauth operand bundles.
virtual bool supportSwiftError() const
Return true if the target supports swifterror attribute.
virtual SDValue visitMaskedLoad(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue &NewLoad, SDValue Ptr, SDValue PassThru, SDValue Mask) const
virtual SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType) const
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::vector< AsmOperandInfo > AsmOperandInfoVector
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
virtual SDValue prepareVolatileOrAtomicLoad(SDValue Chain, const SDLoc &DL, SelectionDAG &DAG) const
This callback is used to prepare for a volatile or atomic load.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const
Target-specific splitting of values into parts that fit a register storing a legal type.
virtual SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const
Target-specific combining of register parts into its original value.
virtual SDValue LowerCall(CallLoweringInfo &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower calls into the specified DAG.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Glue, const SDLoc &DL, const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
virtual SDValue LowerFormalArguments(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::InputArg > &, const SDLoc &, SelectionDAG &, SmallVectorImpl< SDValue > &) const
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
virtual AsmOperandInfoVector ParseConstraints(const DataLayout &DL, const TargetRegisterInfo *TRI, const CallBase &Call) const
Split up the constraint string from the inline assembly value into the specific constraints and their...
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
virtual SDValue LowerReturn(SDValue, CallingConv::ID, bool, const SmallVectorImpl< ISD::OutputArg > &, const SmallVectorImpl< SDValue > &, const SDLoc &, SelectionDAG &) const
This hook must be implemented to lower outgoing return values, described by the Outs array,...
virtual bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const
For some targets, an LLVM struct type must be broken down into multiple simple types,...
virtual void ComputeConstraintToUse(AsmOperandInfo &OpInfo, SDValue Op, SelectionDAG *DAG=nullptr) const
Determines the constraint code and constraint type to use for the specific AsmOperandInfo,...
virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) const
virtual SDValue visitMaskedStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, MachineMemOperand *MMO, SDValue Ptr, SDValue Val, SDValue Mask) const
virtual bool useLoadStackGuardNode(const Module &M) const
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual void LowerOperationWrapper(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const
This callback is invoked by the type legalizer to legalize nodes with an illegal operand type but leg...
virtual bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
virtual MVT getJumpTableRegTy(const DataLayout &DL) const
virtual bool CanLowerReturn(CallingConv::ID, MachineFunction &, bool, const SmallVectorImpl< ISD::OutputArg > &, LLVMContext &, const Type *RetTy) const
This hook should be implemented to check whether the return values described by the Outs array can fi...
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
virtual const TargetIntrinsicInfo * getIntrinsicInfo() const
If intrinsic information is available, return it. If not, return null.
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
virtual TargetTransformInfo getTargetTransformInfo(const Function &F) const
Return a TargetTransformInfo for a given function.
TargetOptions Options
CodeModel::Model getCodeModel() const
Returns the code model.
unsigned NoNaNsFPMath
NoNaNsFPMath - This flag is enabled when the -enable-no-nans-fp-math flag is specified on the command...
unsigned EnableFastISel
EnableFastISel - This flag enables fast-path instruction selection which trades away generated code q...
unsigned NoTrapAfterNoreturn
Do not emit a trap instruction for 'unreachable' IR instructions behind noreturn calls,...
unsigned TrapUnreachable
Emit target-specific trap instruction for 'unreachable' IR instructions.
FPOpFusion::FPOpFusionMode AllowFPOpFusion
AllowFPOpFusion - This flag is set by the -fp-contract=xxx option.
unsigned getID() const
Return the register class ID number.
iterator begin() const
begin/end - Return all of the registers in this class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetInstrInfo * getInstrInfo() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
@ TCK_Latency
The latency of instruction.
InstructionCost getInstructionCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
bool hasConditionalLoadStoreForType(Type *Ty=nullptr) const
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:395
bool isAArch64() const
Tests whether the target is AArch64 (little and big endian).
Definition: Triple.h:948
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:345
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:270
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:264
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:237
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:234
bool isFPOrFPVectorTy() const
Return true if this is a FP type or a vector of FP.
Definition: Type.h:225
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:355
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1859
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:228
This class represents the va_arg llvm instruction, which returns an argument of the specified type gi...
CmpInst::Predicate getPredicate() const
This is the common base class for vector predication intrinsics.
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
MaybeAlign getPointerAlignment() const
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
iterator_range< user_iterator > users()
Definition: Value.h:421
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
Base class of all SIMD vector types.
Definition: DerivedTypes.h:427
Type * getElementType() const
Definition: DerivedTypes.h:460
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:202
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
const ParentTy * getParent() const
Definition: ilist_node.h:32
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: Lint.cpp:87
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:125
@ AnyReg
OBSOLETED - Used for stack based JavaScript calls.
Definition: CallingConv.h:60
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
Definition: CallingConv.h:245
@ X86_VectorCall
MSVC calling convention that passes vectors and vector aggregates in SSE registers.
Definition: CallingConv.h:163
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:243
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1197
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1193
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:753
@ CONVERGENCECTRL_ANCHOR
Definition: ISDOpcodes.h:1470
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:491
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1347
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:1069
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1417
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1450
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:153
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1340
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:574
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:744
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:374
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1226
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1342
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1312
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1343
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1073
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:246
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:380
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1092
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:814
@ ATOMIC_LOAD_USUB_COND
Definition: ISDOpcodes.h:1351
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:498
@ VECTOR_FIND_LAST_ACTIVE
Definition: ISDOpcodes.h:1485
@ FATAN2
FATAN2 - atan2, inspired by libm.
Definition: ISDOpcodes.h:999
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:205
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:157
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1325
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:841
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:558
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1435
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:397
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1439
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:717
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1304
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1096
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1449
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:492
@ FPTRUNC_ROUND
FPTRUNC_ROUND - This corresponds to the fptrunc_round intrinsic.
Definition: ISDOpcodes.h:495
@ FAKE_USE
FAKE_USE represents a use of the operand but does not do anything.
Definition: ISDOpcodes.h:1383
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1338
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:954
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:236
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1339
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1270
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:997
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:387
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1345
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1173
@ ATOMIC_LOAD_USUB_SAT
Definition: ISDOpcodes.h:1352
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:141
@ ANNOTATION_LABEL
ANNOTATION_LABEL - Represents a mid basic block label used by annotations.
Definition: ISDOpcodes.h:1179
@ SET_ROUNDING
Set rounding mode.
Definition: ISDOpcodes.h:936
@ CONVERGENCECTRL_GLUE
Definition: ISDOpcodes.h:1476
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:805
@ PREALLOCATED_SETUP
PREALLOCATED_SETUP - This has 2 operands: an input chain and a SRCVALUE with the preallocated call Va...
Definition: ISDOpcodes.h:1231
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1259
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:107
@ CONVERGENCECTRL_ENTRY
Definition: ISDOpcodes.h:1471
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:1118
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1432
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:752
@ WRITE_REGISTER
Definition: ISDOpcodes.h:125
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1292
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1436
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:1059
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1346
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:334
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1341
@ PREALLOCATED_ARG
PREALLOCATED_ARG - This has 3 operands: an input chain, a SRCVALUE with the preallocated call Value,...
Definition: ISDOpcodes.h:1234
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:1123
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1127
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
Definition: ISDOpcodes.h:601
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:515
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:522
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:356
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:757
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1308
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1451
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:229
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:642
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1222
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1348
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:215
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:330
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1296
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1444
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:931
@ CLEANUPRET
CLEANUPRET - Represents a return from a cleanup block funclet.
Definition: ISDOpcodes.h:1188
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1087
@ GET_FPENV
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1064
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:735
@ PtrAuthGlobalAddress
A ptrauth constant.
Definition: ISDOpcodes.h:90
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1336
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:588
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:124
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:550
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:811
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1282
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:907
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1407
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1344
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
Definition: ISDOpcodes.h:120
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:1031
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1286
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:366
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:338
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1112
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:697
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
Definition: ISDOpcodes.h:606
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:393
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:939
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:766
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1452
@ PCMARKER
PCMARKER - This corresponds to the pcmarker intrinsic.
Definition: ISDOpcodes.h:1245
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
Definition: ISDOpcodes.h:1168
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition: ISDOpcodes.h:135
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:100
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1350
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1334
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:480
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:1050
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1335
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:887
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1253
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:709
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1279
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:190
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
Definition: ISDOpcodes.h:705
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1433
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:407
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition: ISDOpcodes.h:223
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:539
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:627
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1333
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:1004
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:920
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
Definition: ISDOpcodes.h:669
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition: ISDOpcodes.h:112
@ CONVERGENCECTRL_LOOP
Definition: ISDOpcodes.h:1472
@ INLINEASM
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:1165
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:906
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1440
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:147
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:817
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1141
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1418
@ CATCHRET
CATCHRET - Represents a return from a catch block funclet.
Definition: ISDOpcodes.h:1184
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1349
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:508
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:347
@ AssertZext
Definition: ISDOpcodes.h:62
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
Definition: ISDOpcodes.h:595
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1398
@ FMINIMUMNUM
FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with FMINNUM_IEEE and FMAXNUM_IEEE besid...
Definition: ISDOpcodes.h:1055
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1276
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:198
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:530
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1572
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1610
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:885
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
VScaleVal_match m_VScale()
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Offsets
Offsets in bytes from the start of the input buffer.
Definition: SIInstrInfo.h:1613
std::vector< CaseCluster > CaseClusterVector
void sortAndRangeify(CaseClusterVector &Clusters)
Sort Clusters and merge adjacent cases.
CaseClusterVector::iterator CaseClusterIt
std::pair< JumpTableHeader, JumpTable > JumpTableBlock
@ CC_Range
A cluster of adjacent case labels with the same destination, or just one case.
@ CC_JumpTable
A cluster of cases suitable for jump table lowering.
@ CC_BitTests
A cluster of cases suitable for bit test lowering.
Reg
All possible values of the reg field in the ModR/M byte.
@ ReallyHidden
Definition: CommandLine.h:138
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:463
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:147
ExceptionBehavior
Exception behavior used for floating point operations.
Definition: FPEnv.h:38
@ ebStrict
This corresponds to "fpexcept.strict".
Definition: FPEnv.h:41
@ ebMayTrap
This corresponds to "fpexcept.maytrap".
Definition: FPEnv.h:40
@ ebIgnore
This corresponds to "fpexcept.ignore".
Definition: FPEnv.h:39
constexpr float log2ef
Definition: MathExtras.h:67
constexpr double e
Definition: MathExtras.h:48
constexpr float ln2f
Definition: MathExtras.h:65
NodeAddr< FuncNode * > Func
Definition: RDFGraph.h:393
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:355
@ Offset
Definition: DWP.cpp:480
@ Length
Definition: DWP.cpp:480
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1739
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1697
void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:257
bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI)
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
@ Done
Definition: Threading.h:61
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
void diagnoseDontCall(const CallInst &CI)
auto successors(const MachineBasicBlock *BB)
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2115
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:557
gep_type_iterator gep_type_end(const User *GEP)
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
bool isScopedEHPersonality(EHPersonality Pers)
Returns true if this personality uses scope-style EH IR instructions: catchswitch,...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1746
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
@ SPF_ABS
Floating point maxnum.
@ SPF_NABS
Absolute value.
@ SPF_FMAXNUM
Floating point minnum.
@ SPF_UMIN
Signed minimum.
@ SPF_UMAX
Signed maximum.
@ SPF_SMAX
Unsigned minimum.
@ SPF_FMINNUM
Unsigned maximum.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
detail::zippy< detail::zip_first, T, U, Args... > zip_first(T &&t, U &&u, Args &&...args)
zip iterator that, for the sake of efficiency, assumes the first iteratee to be the shortest.
Definition: STLExtras.h:877
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1664
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
auto succ_size(const MachineBasicBlock *BB)
bool hasSingleElement(ContainerTy &&C)
Returns true if the given container only contains a single element.
Definition: STLExtras.h:322
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
ISD::CondCode getFCmpCondCode(FCmpInst::Predicate Pred)
getFCmpCondCode - Return the ISD condition code corresponding to the given LLVM IR floating-point con...
Definition: Analysis.cpp:199
EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
Definition: Local.cpp:2617
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Global
Append to llvm.global_dtors.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2299
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
@ SPNB_RETURNS_NAN
NaN behavior not applicable.
@ SPNB_RETURNS_OTHER
Given one NaN input, returns the NaN.
@ SPNB_RETURNS_ANY
Given one NaN input, returns the non-NaN.
bool isInTailCallPosition(const CallBase &Call, const TargetMachine &TM, bool ReturnsFirstArg=false)
Test if the given instruction is in a position to be optimized with a tail-call.
Definition: Analysis.cpp:535
DWARFExpression::Operation Op
ISD::CondCode getFCmpCodeWithoutNaN(ISD::CondCode CC)
getFCmpCodeWithoutNaN - Given an ISD condition code comparing floats, return the equivalent code if w...
Definition: Analysis.cpp:221
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
Definition: Analysis.cpp:79
bool isAsynchronousEHPersonality(EHPersonality Pers)
Returns true if this personality function catches asynchronous exceptions.
std::optional< RoundingMode > convertStrToRoundingMode(StringRef)
Returns a valid RoundingMode enumerator when given a string that is valid as input in constrained int...
Definition: FPEnv.cpp:24
gep_type_iterator gep_type_begin(const User *GEP)
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:2099
GlobalValue * ExtractTypeInfo(Value *V)
ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
Definition: Analysis.cpp:177
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, const LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition: STLExtras.h:2087
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, APInt Offset, const DataLayout &DL)
Return the value that a load from C with offset Offset would produce if it is constant and determinab...
unsigned ComputeLinearIndex(Type *Ty, const unsigned *Indices, const unsigned *IndicesEnd, unsigned CurIndex=0)
Compute the linearized index of a member in a nested aggregate/struct/array.
Definition: Analysis.cpp:33
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition: bit.h:327
@ Default
The result values are uniform if and only if all operands are uniform.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
#define NC
Definition: regutils.h:42
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:764
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:257
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:390
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:74
uint64_t getScalarStoreSize() const
Definition: ValueTypes.h:397
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:279
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:295
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:147
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:345
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:368
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:354
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:380
static EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:289
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:311
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
bool isRISCVVectorTuple() const
Return true if this is a vector value type.
Definition: ValueTypes.h:179
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition: ValueTypes.h:376
bool isFixedLengthVector() const
Definition: ValueTypes.h:181
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:168
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:318
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:287
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:210
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:174
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:323
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:157
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition: ValueTypes.h:102
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:331
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:448
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:152
void setPointerAddrSpace(unsigned AS)
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
static const unsigned NoArgIndex
Sentinel value for implicit machine-level input arguments.
OutputArg - This struct carries flags and a value for a single outgoing (actual) argument or outgoing...
ConstraintPrefix Type
Type - The basic type of the constraint: input/output/clobber/label.
Definition: InlineAsm.h:126
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:240
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
Definition: Alignment.h:141
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
SmallVector< std::pair< Register, TypeSize >, 4 > getRegsAndSizes() const
Return a list of registers and their sizes.
RegsForValue()=default
SmallVector< unsigned, 4 > RegCount
This list holds the number of registers for each value.
SmallVector< EVT, 4 > ValueVTs
The value types of the values, which may not be legal, and may need be promoted or synthesized from o...
SmallVector< Register, 4 > Regs
This list holds the registers assigned to the values.
void AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, unsigned MatchingIdx, const SDLoc &dl, SelectionDAG &DAG, std::vector< SDValue > &Ops) const
Add this value to the specified inlineasm node operand list.
SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr) const
Emit a series of CopyFromReg nodes that copies from this value and returns the result as a ValueVTs v...
SmallVector< MVT, 4 > RegVTs
The value types of the registers.
void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const
Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...
std::optional< CallingConv::ID > CallConv
Records if this value needs to be treated in an ABI dependant manner, different to normal type legali...
bool occupiesMultipleRegs() const
Check if the total RegCount is greater than one.
These are IR-level optimization flags that may be propagated to SDNodes.
void copyFMF(const FPMathOperator &FPMO)
Propagate the fast-math-flags from an IR FPMathOperator.
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
A MapVector that performs no allocations if smaller than a certain size.
Definition: MapVector.h:254
This structure is used to communicate between SelectionDAGBuilder and SDISel for the code generation ...
SDLoc DL
The debug location of the instruction this CaseBlock was produced from.
A cluster of case labels.
static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, MachineBasicBlock *MBB, BranchProbability Prob)
This contains information for each constraint that we are lowering.
TargetLowering::ConstraintType ConstraintType
Information about the constraint code, e.g.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setConvergent(bool Value=true)
CallLoweringInfo & setCFIType(const ConstantInt *Type)
SmallVector< ISD::InputArg, 32 > Ins
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setIsPatchPoint(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setIsPreallocated(bool Value=true)
CallLoweringInfo & setConvergenceControlToken(SDValue Token)
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
CallLoweringInfo & setChain(SDValue InChain)
CallLoweringInfo & setPtrAuth(PtrAuthInfo Value)
CallLoweringInfo & setCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList, AttributeSet ResultAttrs={})
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setDiscardResult(bool Value=true)
This structure contains the information necessary for lowering pointer-authenticating indirect calls.
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)