LLVM 21.0.0git
RISCVFrameLowering.cpp
Go to the documentation of this file.
1//===-- RISCVFrameLowering.cpp - RISC-V Frame Information -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of TargetFrameLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVFrameLowering.h"
15#include "RISCVSubtarget.h"
24#include "llvm/MC/MCDwarf.h"
25#include "llvm/Support/LEB128.h"
26
27#include <algorithm>
28
29using namespace llvm;
30
31namespace {
32
33class CFISaveRegisterEmitter {
36
37public:
38 CFISaveRegisterEmitter(MachineFunction &MF)
39 : MF{MF}, MFI{MF.getFrameInfo()} {};
40
42 const RISCVRegisterInfo &RI, const RISCVInstrInfo &TII,
43 const DebugLoc &DL, const CalleeSavedInfo &CS) const {
44 int FrameIdx = CS.getFrameIdx();
45 int64_t Offset = MFI.getObjectOffset(FrameIdx);
46 Register Reg = CS.getReg();
47 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset(
48 nullptr, RI.getDwarfRegNum(Reg, true), Offset));
49 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
50 .addCFIIndex(CFIIndex)
52 }
53};
54
55class CFIRestoreRegisterEmitter {
57
58public:
59 CFIRestoreRegisterEmitter(MachineFunction &MF) : MF{MF} {};
60
62 const RISCVRegisterInfo &RI, const RISCVInstrInfo &TII,
63 const DebugLoc &DL, const CalleeSavedInfo &CS) const {
64 Register Reg = CS.getReg();
65 unsigned CFIIndex = MF.addFrameInst(
66 MCCFIInstruction::createRestore(nullptr, RI.getDwarfRegNum(Reg, true)));
67 BuildMI(MBB, MBBI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
68 .addCFIIndex(CFIIndex)
70 }
71};
72
73} // namespace
74
75template <typename Emitter>
76void RISCVFrameLowering::emitCFIForCSI(
78 const SmallVector<CalleeSavedInfo, 8> &CSI) const {
83
84 Emitter E{*MF};
85 for (const auto &CS : CSI)
86 E.emit(MBB, MBBI, *RI, *TII, DL, CS);
87}
88
90 if (ABI == RISCVABI::ABI_ILP32E)
91 return Align(4);
92 if (ABI == RISCVABI::ABI_LP64E)
93 return Align(8);
94 return Align(16);
95}
96
99 StackGrowsDown, getABIStackAlignment(STI.getTargetABI()),
100 /*LocalAreaOffset=*/0,
101 /*TransientStackAlignment=*/getABIStackAlignment(STI.getTargetABI())),
102 STI(STI) {}
103
104// The register used to hold the frame pointer.
105static constexpr Register FPReg = RISCV::X8;
106
107// The register used to hold the stack pointer.
108static constexpr Register SPReg = RISCV::X2;
109
110// The register used to hold the return address.
111static constexpr Register RAReg = RISCV::X1;
112
113// Offsets which need to be scale by XLen representing locations of CSRs which
114// are given a fixed location by save/restore libcalls or Zcmp Push/Pop.
115static const std::pair<MCPhysReg, int8_t> FixedCSRFIMap[] = {
116 {/*ra*/ RAReg, -1}, {/*s0*/ FPReg, -2},
117 {/*s1*/ RISCV::X9, -3}, {/*s2*/ RISCV::X18, -4},
118 {/*s3*/ RISCV::X19, -5}, {/*s4*/ RISCV::X20, -6},
119 {/*s5*/ RISCV::X21, -7}, {/*s6*/ RISCV::X22, -8},
120 {/*s7*/ RISCV::X23, -9}, {/*s8*/ RISCV::X24, -10},
121 {/*s9*/ RISCV::X25, -11}, {/*s10*/ RISCV::X26, -12},
122 {/*s11*/ RISCV::X27, -13}};
123
124// For now we use x3, a.k.a gp, as pointer to shadow call stack.
125// User should not use x3 in their asm.
128 const DebugLoc &DL) {
129 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
130 bool HasHWShadowStack = MF.getFunction().hasFnAttribute("hw-shadow-stack") &&
131 STI.hasStdExtZicfiss();
132 bool HasSWShadowStack =
133 MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack);
134 if (!HasHWShadowStack && !HasSWShadowStack)
135 return;
136
137 const llvm::RISCVRegisterInfo *TRI = STI.getRegisterInfo();
138 Register RAReg = TRI->getRARegister();
139
140 // Do not save RA to the SCS if it's not saved to the regular stack,
141 // i.e. RA is not at risk of being overwritten.
142 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
143 if (llvm::none_of(
144 CSI, [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
145 return;
146
147 const RISCVInstrInfo *TII = STI.getInstrInfo();
148 if (HasHWShadowStack) {
149 BuildMI(MBB, MI, DL, TII->get(RISCV::SSPUSH)).addReg(RAReg);
150 return;
151 }
152
153 Register SCSPReg = RISCVABI::getSCSPReg();
154
155 bool IsRV64 = STI.is64Bit();
156 int64_t SlotSize = STI.getXLen() / 8;
157 // Store return address to shadow call stack
158 // addi gp, gp, [4|8]
159 // s[w|d] ra, -[4|8](gp)
160 BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
161 .addReg(SCSPReg, RegState::Define)
162 .addReg(SCSPReg)
163 .addImm(SlotSize)
165 BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
166 .addReg(RAReg)
167 .addReg(SCSPReg)
168 .addImm(-SlotSize)
170
171 // Emit a CFI instruction that causes SlotSize to be subtracted from the value
172 // of the shadow stack pointer when unwinding past this frame.
173 char DwarfSCSReg = TRI->getDwarfRegNum(SCSPReg, /*IsEH*/ true);
174 assert(DwarfSCSReg < 32 && "SCS Register should be < 32 (X3).");
175
176 char Offset = static_cast<char>(-SlotSize) & 0x7f;
177 const char CFIInst[] = {
178 dwarf::DW_CFA_val_expression,
179 DwarfSCSReg, // register
180 2, // length
181 static_cast<char>(unsigned(dwarf::DW_OP_breg0 + DwarfSCSReg)),
182 Offset, // addend (sleb128)
183 };
184
185 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
186 nullptr, StringRef(CFIInst, sizeof(CFIInst))));
187 BuildMI(MBB, MI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
188 .addCFIIndex(CFIIndex)
190}
191
194 const DebugLoc &DL) {
195 const auto &STI = MF.getSubtarget<RISCVSubtarget>();
196 bool HasHWShadowStack = MF.getFunction().hasFnAttribute("hw-shadow-stack") &&
197 STI.hasStdExtZicfiss();
198 bool HasSWShadowStack =
199 MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack);
200 if (!HasHWShadowStack && !HasSWShadowStack)
201 return;
202
203 Register RAReg = STI.getRegisterInfo()->getRARegister();
204
205 // See emitSCSPrologue() above.
206 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
207 if (llvm::none_of(
208 CSI, [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; }))
209 return;
210
211 const RISCVInstrInfo *TII = STI.getInstrInfo();
212 if (HasHWShadowStack) {
213 BuildMI(MBB, MI, DL, TII->get(RISCV::SSPOPCHK)).addReg(RAReg);
214 return;
215 }
216
217 Register SCSPReg = RISCVABI::getSCSPReg();
218
219 bool IsRV64 = STI.is64Bit();
220 int64_t SlotSize = STI.getXLen() / 8;
221 // Load return address from shadow call stack
222 // l[w|d] ra, -[4|8](gp)
223 // addi gp, gp, -[4|8]
224 BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::LD : RISCV::LW))
226 .addReg(SCSPReg)
227 .addImm(-SlotSize)
229 BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI))
230 .addReg(SCSPReg, RegState::Define)
231 .addReg(SCSPReg)
232 .addImm(-SlotSize)
234 // Restore the SCS pointer
235 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(
236 nullptr, STI.getRegisterInfo()->getDwarfRegNum(SCSPReg, /*IsEH*/ true)));
237 BuildMI(MBB, MI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
238 .addCFIIndex(CFIIndex)
240}
241
242// Get the ID of the libcall used for spilling and restoring callee saved
243// registers. The ID is representative of the number of registers saved or
244// restored by the libcall, except it is zero-indexed - ID 0 corresponds to a
245// single register.
246static int getLibCallID(const MachineFunction &MF,
247 const std::vector<CalleeSavedInfo> &CSI) {
248 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
249
250 if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF))
251 return -1;
252
253 Register MaxReg = RISCV::NoRegister;
254 for (auto &CS : CSI)
255 // assignCalleeSavedSpillSlots assigns negative frame indexes to
256 // registers which can be saved by libcall.
257 if (CS.getFrameIdx() < 0)
258 MaxReg = std::max(MaxReg.id(), CS.getReg().id());
259
260 if (MaxReg == RISCV::NoRegister)
261 return -1;
262
263 switch (MaxReg) {
264 default:
265 llvm_unreachable("Something has gone wrong!");
266 // clang-format off
267 case /*s11*/ RISCV::X27: return 12;
268 case /*s10*/ RISCV::X26: return 11;
269 case /*s9*/ RISCV::X25: return 10;
270 case /*s8*/ RISCV::X24: return 9;
271 case /*s7*/ RISCV::X23: return 8;
272 case /*s6*/ RISCV::X22: return 7;
273 case /*s5*/ RISCV::X21: return 6;
274 case /*s4*/ RISCV::X20: return 5;
275 case /*s3*/ RISCV::X19: return 4;
276 case /*s2*/ RISCV::X18: return 3;
277 case /*s1*/ RISCV::X9: return 2;
278 case /*s0*/ FPReg: return 1;
279 case /*ra*/ RAReg: return 0;
280 // clang-format on
281 }
282}
283
284// Get the name of the libcall used for spilling callee saved registers.
285// If this function will not use save/restore libcalls, then return a nullptr.
286static const char *
288 const std::vector<CalleeSavedInfo> &CSI) {
289 static const char *const SpillLibCalls[] = {
290 "__riscv_save_0",
291 "__riscv_save_1",
292 "__riscv_save_2",
293 "__riscv_save_3",
294 "__riscv_save_4",
295 "__riscv_save_5",
296 "__riscv_save_6",
297 "__riscv_save_7",
298 "__riscv_save_8",
299 "__riscv_save_9",
300 "__riscv_save_10",
301 "__riscv_save_11",
302 "__riscv_save_12"
303 };
304
305 int LibCallID = getLibCallID(MF, CSI);
306 if (LibCallID == -1)
307 return nullptr;
308 return SpillLibCalls[LibCallID];
309}
310
311// Get the name of the libcall used for restoring callee saved registers.
312// If this function will not use save/restore libcalls, then return a nullptr.
313static const char *
315 const std::vector<CalleeSavedInfo> &CSI) {
316 static const char *const RestoreLibCalls[] = {
317 "__riscv_restore_0",
318 "__riscv_restore_1",
319 "__riscv_restore_2",
320 "__riscv_restore_3",
321 "__riscv_restore_4",
322 "__riscv_restore_5",
323 "__riscv_restore_6",
324 "__riscv_restore_7",
325 "__riscv_restore_8",
326 "__riscv_restore_9",
327 "__riscv_restore_10",
328 "__riscv_restore_11",
329 "__riscv_restore_12"
330 };
331
332 int LibCallID = getLibCallID(MF, CSI);
333 if (LibCallID == -1)
334 return nullptr;
335 return RestoreLibCalls[LibCallID];
336}
337
338// Return encoded value and register count for PUSH/POP instruction,
339// representing registers to store/load.
340static std::pair<unsigned, unsigned>
342 switch (MaxReg) {
343 default:
344 llvm_unreachable("Unexpected Reg for Push/Pop Inst");
345 case RISCV::X27: /*s11*/
346 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S11, 13);
347 case RISCV::X25: /*s9*/
348 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S9, 11);
349 case RISCV::X24: /*s8*/
350 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S8, 10);
351 case RISCV::X23: /*s7*/
352 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S7, 9);
353 case RISCV::X22: /*s6*/
354 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S6, 8);
355 case RISCV::X21: /*s5*/
356 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S5, 7);
357 case RISCV::X20: /*s4*/
358 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S4, 6);
359 case RISCV::X19: /*s3*/
360 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S3, 5);
361 case RISCV::X18: /*s2*/
362 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S2, 4);
363 case RISCV::X9: /*s1*/
364 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0_S1, 3);
365 case FPReg: /*s0*/
366 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA_S0, 2);
367 case RAReg: /*ra*/
368 return std::make_pair(llvm::RISCVZC::RLISTENCODE::RA, 1);
369 }
370}
371
372// Get the max reg of Push/Pop for restoring callee saved registers.
374 const std::vector<CalleeSavedInfo> &CSI) {
375 Register MaxPushPopReg = RISCV::NoRegister;
376 for (auto &CS : CSI) {
377 if (llvm::find_if(FixedCSRFIMap, [&](auto P) {
378 return P.first == CS.getReg();
379 }) != std::end(FixedCSRFIMap))
380 MaxPushPopReg = std::max(MaxPushPopReg.id(), CS.getReg().id());
381 }
382 assert(MaxPushPopReg != RISCV::X26 && "x26 requires x27 to also be pushed");
383 return MaxPushPopReg;
384}
385
386// Return true if the specified function should have a dedicated frame
387// pointer register. This is true if frame pointer elimination is
388// disabled, if it needs dynamic stack realignment, if the function has
389// variable sized allocas, or if the frame address is taken.
391 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
392
393 const MachineFrameInfo &MFI = MF.getFrameInfo();
394 return MF.getTarget().Options.DisableFramePointerElim(MF) ||
395 RegInfo->hasStackRealignment(MF) || MFI.hasVarSizedObjects() ||
397}
398
400 const MachineFrameInfo &MFI = MF.getFrameInfo();
402
403 // If we do not reserve stack space for outgoing arguments in prologue,
404 // we will adjust the stack pointer before call instruction. After the
405 // adjustment, we can not use SP to access the stack objects for the
406 // arguments. Instead, use BP to access these stack objects.
407 return (MFI.hasVarSizedObjects() ||
409 MFI.getMaxCallFrameSize() != 0))) &&
410 TRI->hasStackRealignment(MF);
411}
412
413// Determines the size of the frame and maximum call frame size.
414void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
415 MachineFrameInfo &MFI = MF.getFrameInfo();
416 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
417
418 // Get the number of bytes to allocate from the FrameInfo.
419 uint64_t FrameSize = MFI.getStackSize();
420
421 // Get the alignment.
422 Align StackAlign = getStackAlign();
423
424 // Make sure the frame is aligned.
425 FrameSize = alignTo(FrameSize, StackAlign);
426
427 // Update frame info.
428 MFI.setStackSize(FrameSize);
429
430 // When using SP or BP to access stack objects, we may require extra padding
431 // to ensure the bottom of the RVV stack is correctly aligned within the main
432 // stack. We calculate this as the amount required to align the scalar local
433 // variable section up to the RVV alignment.
435 if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) {
436 int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() -
437 RVFI->getVarArgsSaveSize();
438 if (auto RVVPadding =
439 offsetToAlignment(ScalarLocalVarSize, RVFI->getRVVStackAlign()))
440 RVFI->setRVVPadding(RVVPadding);
441 }
442}
443
444// Returns the stack size including RVV padding (when required), rounded back
445// up to the required stack alignment.
447 const MachineFunction &MF) const {
448 const MachineFrameInfo &MFI = MF.getFrameInfo();
449 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
450 return alignTo(MFI.getStackSize() + RVFI->getRVVPadding(), getStackAlign());
451}
452
455 const std::vector<CalleeSavedInfo> &CSI) {
456 const MachineFrameInfo &MFI = MF.getFrameInfo();
458
459 for (auto &CS : CSI) {
460 int FI = CS.getFrameIdx();
461 if (FI >= 0 && MFI.getStackID(FI) == TargetStackID::Default)
462 NonLibcallCSI.push_back(CS);
463 }
464
465 return NonLibcallCSI;
466}
467
470 const std::vector<CalleeSavedInfo> &CSI) {
471 const MachineFrameInfo &MFI = MF.getFrameInfo();
473
474 for (auto &CS : CSI) {
475 int FI = CS.getFrameIdx();
476 if (FI >= 0 && MFI.getStackID(FI) == TargetStackID::ScalableVector)
477 RVVCSI.push_back(CS);
478 }
479
480 return RVVCSI;
481}
482
485 const std::vector<CalleeSavedInfo> &CSI) {
486 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
487
488 SmallVector<CalleeSavedInfo, 8> PushOrLibCallsCSI;
489 if (!RVFI->useSaveRestoreLibCalls(MF) && !RVFI->isPushable(MF))
490 return PushOrLibCallsCSI;
491
492 for (const auto &CS : CSI) {
493 const auto *FII = llvm::find_if(
494 FixedCSRFIMap, [&](auto P) { return P.first == CS.getReg(); });
495 if (FII != std::end(FixedCSRFIMap))
496 PushOrLibCallsCSI.push_back(CS);
497 }
498
499 return PushOrLibCallsCSI;
500}
501
502void RISCVFrameLowering::allocateAndProbeStackForRVV(
504 MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t Amount,
505 MachineInstr::MIFlag Flag, bool EmitCFI, bool DynAllocation) const {
506 assert(Amount != 0 && "Did not need to adjust stack pointer for RVV.");
507
508 // Emit a variable-length allocation probing loop.
509
510 // Get VLEN in TargetReg
512 Register TargetReg = RISCV::X6;
513 uint32_t NumOfVReg = Amount / (RISCV::RVVBitsPerBlock / 8);
514 BuildMI(MBB, MBBI, DL, TII->get(RISCV::PseudoReadVLENB), TargetReg)
515 .setMIFlag(Flag);
516 TII->mulImm(MF, MBB, MBBI, DL, TargetReg, NumOfVReg, Flag);
517
518 if (EmitCFI) {
519 // Set the CFA register to TargetReg.
520 unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(TargetReg, true);
521 unsigned CFIIndex =
522 MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, -Amount));
523 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
524 .addCFIIndex(CFIIndex)
526 }
527
528 // It will be expanded to a probe loop in `inlineStackProbe`.
529 BuildMI(MBB, MBBI, DL, TII->get(RISCV::PROBED_STACKALLOC_RVV))
530 .addReg(SPReg)
531 .addReg(TargetReg);
532
533 if (EmitCFI) {
534 // Set the CFA register back to SP.
535 unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(SPReg, true);
536 unsigned CFIIndex =
538 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
539 .addCFIIndex(CFIIndex)
541 }
542
543 // SUB SP, SP, T1
544 BuildMI(MBB, MBBI, DL, TII->get(RISCV::SUB), SPReg)
545 .addReg(SPReg)
546 .addReg(TargetReg)
547 .setMIFlag(Flag);
548
549 // If we have a dynamic allocation later we need to probe any residuals.
550 if (DynAllocation) {
551 BuildMI(MBB, MBBI, DL, TII->get(STI.is64Bit() ? RISCV::SD : RISCV::SW))
552 .addReg(RISCV::X0)
553 .addReg(SPReg)
554 .addImm(0)
556 }
557}
558
561 int FixedOffset, int ScalableOffset,
562 llvm::raw_string_ostream &Comment) {
563 unsigned DwarfVLenB = TRI.getDwarfRegNum(RISCV::VLENB, true);
564 uint8_t Buffer[16];
565 if (FixedOffset) {
566 Expr.push_back(dwarf::DW_OP_consts);
567 Expr.append(Buffer, Buffer + encodeSLEB128(FixedOffset, Buffer));
568 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
569 Comment << (FixedOffset < 0 ? " - " : " + ") << std::abs(FixedOffset);
570 }
571
572 Expr.push_back((uint8_t)dwarf::DW_OP_consts);
573 Expr.append(Buffer, Buffer + encodeSLEB128(ScalableOffset, Buffer));
574
575 Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
576 Expr.append(Buffer, Buffer + encodeULEB128(DwarfVLenB, Buffer));
577 Expr.push_back(0);
578
579 Expr.push_back((uint8_t)dwarf::DW_OP_mul);
580 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
581
582 Comment << (ScalableOffset < 0 ? " - " : " + ") << std::abs(ScalableOffset)
583 << " * vlenb";
584}
585
587 Register Reg,
588 uint64_t FixedOffset,
589 uint64_t ScalableOffset) {
590 assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
591 SmallString<64> Expr;
592 std::string CommentBuffer;
593 llvm::raw_string_ostream Comment(CommentBuffer);
594 // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB).
595 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
596 Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
597 Expr.push_back(0);
598 if (Reg == SPReg)
599 Comment << "sp";
600 else
601 Comment << printReg(Reg, &TRI);
602
603 appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset,
604 Comment);
605
606 SmallString<64> DefCfaExpr;
607 uint8_t Buffer[16];
608 DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
609 DefCfaExpr.append(Buffer, Buffer + encodeULEB128(Expr.size(), Buffer));
610 DefCfaExpr.append(Expr.str());
611
612 return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(), SMLoc(),
613 Comment.str());
614}
615
617 Register Reg, uint64_t FixedOffset,
618 uint64_t ScalableOffset) {
619 assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV");
620 SmallString<64> Expr;
621 std::string CommentBuffer;
622 llvm::raw_string_ostream Comment(CommentBuffer);
623 Comment << printReg(Reg, &TRI) << " @ cfa";
624
625 // Build up the expression (FixedOffset + ScalableOffset * VLENB).
626 appendScalableVectorExpression(TRI, Expr, FixedOffset, ScalableOffset,
627 Comment);
628
629 SmallString<64> DefCfaExpr;
630 uint8_t Buffer[16];
631 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
632 DefCfaExpr.push_back(dwarf::DW_CFA_expression);
633 DefCfaExpr.append(Buffer, Buffer + encodeULEB128(DwarfReg, Buffer));
634 DefCfaExpr.append(Buffer, Buffer + encodeULEB128(Expr.size(), Buffer));
635 DefCfaExpr.append(Expr.str());
636
637 return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(), SMLoc(),
638 Comment.str());
639}
640
641// Allocate stack space and probe it if necessary.
645 uint64_t RealStackSize, bool EmitCFI,
646 bool NeedProbe, uint64_t ProbeSize,
647 bool DynAllocation) const {
648 DebugLoc DL;
651 bool IsRV64 = STI.is64Bit();
652
653 // Simply allocate the stack if it's not big enough to require a probe.
654 if (!NeedProbe || Offset <= ProbeSize) {
657
658 if (EmitCFI) {
659 // Emit ".cfi_def_cfa_offset RealStackSize"
660 unsigned CFIIndex = MF.addFrameInst(
661 MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize));
662 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
663 .addCFIIndex(CFIIndex)
665 }
666
667 if (NeedProbe && DynAllocation) {
668 // s[d|w] zero, 0(sp)
669 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
670 .addReg(RISCV::X0)
671 .addReg(SPReg)
672 .addImm(0)
674 }
675
676 return;
677 }
678
679 // Unroll the probe loop depending on the number of iterations.
680 if (Offset < ProbeSize * 5) {
681 uint64_t CurrentOffset = 0;
682 while (CurrentOffset + ProbeSize <= Offset) {
683 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
685 getStackAlign());
686 // s[d|w] zero, 0(sp)
687 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
688 .addReg(RISCV::X0)
689 .addReg(SPReg)
690 .addImm(0)
692
693 CurrentOffset += ProbeSize;
694 if (EmitCFI) {
695 // Emit ".cfi_def_cfa_offset CurrentOffset"
696 unsigned CFIIndex = MF.addFrameInst(
697 MCCFIInstruction::cfiDefCfaOffset(nullptr, CurrentOffset));
698 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
699 .addCFIIndex(CFIIndex)
701 }
702 }
703
704 uint64_t Residual = Offset - CurrentOffset;
705 if (Residual) {
706 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
708 getStackAlign());
709 if (EmitCFI) {
710 // Emit ".cfi_def_cfa_offset Offset"
711 unsigned CFIIndex =
713 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
714 .addCFIIndex(CFIIndex)
716 }
717
718 if (DynAllocation) {
719 // s[d|w] zero, 0(sp)
720 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
721 .addReg(RISCV::X0)
722 .addReg(SPReg)
723 .addImm(0)
725 }
726 }
727
728 return;
729 }
730
731 // Emit a variable-length allocation probing loop.
732 uint64_t RoundedSize = alignDown(Offset, ProbeSize);
733 uint64_t Residual = Offset - RoundedSize;
734
735 Register TargetReg = RISCV::X6;
736 // SUB TargetReg, SP, RoundedSize
737 RI->adjustReg(MBB, MBBI, DL, TargetReg, SPReg,
739 getStackAlign());
740
741 if (EmitCFI) {
742 // Set the CFA register to TargetReg.
743 unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(TargetReg, true);
744 unsigned CFIIndex =
745 MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, RoundedSize));
746 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
747 .addCFIIndex(CFIIndex)
749 }
750
751 // It will be expanded to a probe loop in `inlineStackProbe`.
752 BuildMI(MBB, MBBI, DL, TII->get(RISCV::PROBED_STACKALLOC))
753 .addReg(SPReg)
754 .addReg(TargetReg);
755
756 if (EmitCFI) {
757 // Set the CFA register back to SP.
758 unsigned Reg = STI.getRegisterInfo()->getDwarfRegNum(SPReg, true);
759 unsigned CFIIndex =
761 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
762 .addCFIIndex(CFIIndex)
764 }
765
766 if (Residual) {
769 if (DynAllocation) {
770 // s[d|w] zero, 0(sp)
771 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
772 .addReg(RISCV::X0)
773 .addReg(SPReg)
774 .addImm(0)
776 }
777 }
778
779 if (EmitCFI) {
780 // Emit ".cfi_def_cfa_offset Offset"
781 unsigned CFIIndex =
783 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
784 .addCFIIndex(CFIIndex)
786 }
787}
788
790 MachineBasicBlock &MBB) const {
791 MachineFrameInfo &MFI = MF.getFrameInfo();
792 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
796
798
799 // Debug location must be unknown since the first debug location is used
800 // to determine the end of the prologue.
801 DebugLoc DL;
802
803 // All calls are tail calls in GHC calling conv, and functions have no
804 // prologue/epilogue.
806 return;
807
808 // Emit prologue for shadow call stack.
809 emitSCSPrologue(MF, MBB, MBBI, DL);
810
811 auto FirstFrameSetup = MBBI;
812
813 // Skip past all callee-saved register spill instructions.
814 while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
815 ++MBBI;
816
817 // Determine the correct frame layout
818 determineFrameLayout(MF);
819
820 const auto &CSI = MFI.getCalleeSavedInfo();
821
822 // Skip to before the spills of scalar callee-saved registers
823 // FIXME: assumes exactly one instruction is used to restore each
824 // callee-saved register.
825 MBBI = std::prev(MBBI, getRVVCalleeSavedInfo(MF, CSI).size() +
826 getUnmanagedCSI(MF, CSI).size());
827
828 // If libcalls are used to spill and restore callee-saved registers, the frame
829 // has two sections; the opaque section managed by the libcalls, and the
830 // section managed by MachineFrameInfo which can also hold callee saved
831 // registers in fixed stack slots, both of which have negative frame indices.
832 // This gets even more complicated when incoming arguments are passed via the
833 // stack, as these too have negative frame indices. An example is detailed
834 // below:
835 //
836 // | incoming arg | <- FI[-3]
837 // | libcallspill |
838 // | calleespill | <- FI[-2]
839 // | calleespill | <- FI[-1]
840 // | this_frame | <- FI[0]
841 //
842 // For negative frame indices, the offset from the frame pointer will differ
843 // depending on which of these groups the frame index applies to.
844 // The following calculates the correct offset knowing the number of callee
845 // saved registers spilt by the two methods.
846 if (int LibCallRegs = getLibCallID(MF, MFI.getCalleeSavedInfo()) + 1) {
847 // Calculate the size of the frame managed by the libcall. The stack
848 // alignment of these libcalls should be the same as how we set it in
849 // getABIStackAlignment.
850 unsigned LibCallFrameSize =
851 alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
852 RVFI->setLibCallStackSize(LibCallFrameSize);
853
854 unsigned CFIIndex = MF.addFrameInst(
855 MCCFIInstruction::cfiDefCfaOffset(nullptr, LibCallFrameSize));
856 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
857 .addCFIIndex(CFIIndex)
859
860 emitCFIForCSI<CFISaveRegisterEmitter>(MBB, MBBI,
862 }
863
864 // FIXME (note copied from Lanai): This appears to be overallocating. Needs
865 // investigation. Get the number of bytes to allocate from the FrameInfo.
866 uint64_t RealStackSize = getStackSizeWithRVVPadding(MF);
867 uint64_t StackSize = RealStackSize - RVFI->getReservedSpillsSize();
868 uint64_t RVVStackSize = RVFI->getRVVStackSize();
869
870 // Early exit if there is no need to allocate on the stack
871 if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0)
872 return;
873
874 // If the stack pointer has been marked as reserved, then produce an error if
875 // the frame requires stack allocation
878 MF.getFunction(), "Stack pointer required, but has been reserved."});
879
880 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
881 // Split the SP adjustment to reduce the offsets of callee saved spill.
882 if (FirstSPAdjustAmount) {
883 StackSize = FirstSPAdjustAmount;
884 RealStackSize = FirstSPAdjustAmount;
885 }
886
887 if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
888 FirstFrameSetup->getOpcode() == RISCV::CM_PUSH) {
889 // Use available stack adjustment in push instruction to allocate additional
890 // stack space. Align the stack size down to a multiple of 16. This is
891 // needed for RVE.
892 // FIXME: Can we increase the stack size to a multiple of 16 instead?
893 uint64_t Spimm =
894 std::min(alignDown(StackSize, 16), static_cast<uint64_t>(48));
895 FirstFrameSetup->getOperand(1).setImm(Spimm);
896 StackSize -= Spimm;
897
898 unsigned CFIIndex = MF.addFrameInst(
899 MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize - StackSize));
900 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
901 .addCFIIndex(CFIIndex)
903
904 emitCFIForCSI<CFISaveRegisterEmitter>(MBB, MBBI,
906 }
907
908 // Allocate space on the stack if necessary.
909 auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
910 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
911 bool NeedProbe = TLI->hasInlineStackProbe(MF);
912 uint64_t ProbeSize = TLI->getStackProbeSize(MF, getStackAlign());
913 bool DynAllocation =
914 MF.getInfo<RISCVMachineFunctionInfo>()->hasDynamicAllocation();
915 if (StackSize != 0)
916 allocateStack(MBB, MBBI, MF, StackSize, RealStackSize, /*EmitCFI=*/true,
917 NeedProbe, ProbeSize, DynAllocation);
918
919 // The frame pointer is callee-saved, and code has been generated for us to
920 // save it to the stack. We need to skip over the storing of callee-saved
921 // registers as the frame pointer must be modified after it has been saved
922 // to the stack, not before.
923 // FIXME: assumes exactly one instruction is used to save each callee-saved
924 // register.
925 std::advance(MBBI, getUnmanagedCSI(MF, CSI).size());
926
927 // Iterate over list of callee-saved registers and emit .cfi_offset
928 // directives.
929 emitCFIForCSI<CFISaveRegisterEmitter>(MBB, MBBI, getUnmanagedCSI(MF, CSI));
930
931 // Generate new FP.
932 if (hasFP(MF)) {
935 MF.getFunction(), "Frame pointer required, but has been reserved."});
936 // The frame pointer does need to be reserved from register allocation.
937 assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved");
938
939 RI->adjustReg(MBB, MBBI, DL, FPReg, SPReg,
940 StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()),
942
943 // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()"
944 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
945 nullptr, RI->getDwarfRegNum(FPReg, true), RVFI->getVarArgsSaveSize()));
946 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
947 .addCFIIndex(CFIIndex)
949 }
950
951 uint64_t SecondSPAdjustAmount = 0;
952 // Emit the second SP adjustment after saving callee saved registers.
953 if (FirstSPAdjustAmount) {
954 SecondSPAdjustAmount = getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
955 assert(SecondSPAdjustAmount > 0 &&
956 "SecondSPAdjustAmount should be greater than zero");
957
958 allocateStack(MBB, MBBI, MF, SecondSPAdjustAmount,
959 getStackSizeWithRVVPadding(MF), !hasFP(MF), NeedProbe,
960 ProbeSize, DynAllocation);
961 }
962
963 if (RVVStackSize) {
964 if (NeedProbe) {
965 allocateAndProbeStackForRVV(MF, MBB, MBBI, DL, RVVStackSize,
967 DynAllocation);
968 } else {
969 // We must keep the stack pointer aligned through any intermediate
970 // updates.
971 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg,
972 StackOffset::getScalable(-RVVStackSize),
974 }
975
976 if (!hasFP(MF)) {
977 // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb".
978 unsigned CFIIndex = MF.addFrameInst(createDefCFAExpression(
979 *RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8));
980 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
981 .addCFIIndex(CFIIndex)
983 }
984
985 std::advance(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
986 emitCalleeSavedRVVPrologCFI(MBB, MBBI, hasFP(MF));
987 }
988
989 if (hasFP(MF)) {
990 // Realign Stack
992 if (RI->hasStackRealignment(MF)) {
993 Align MaxAlignment = MFI.getMaxAlign();
994
996 if (isInt<12>(-(int)MaxAlignment.value())) {
997 BuildMI(MBB, MBBI, DL, TII->get(RISCV::ANDI), SPReg)
998 .addReg(SPReg)
999 .addImm(-(int)MaxAlignment.value())
1001 } else {
1002 unsigned ShiftAmount = Log2(MaxAlignment);
1003 Register VR =
1004 MF.getRegInfo().createVirtualRegister(&RISCV::GPRRegClass);
1005 BuildMI(MBB, MBBI, DL, TII->get(RISCV::SRLI), VR)
1006 .addReg(SPReg)
1007 .addImm(ShiftAmount)
1009 BuildMI(MBB, MBBI, DL, TII->get(RISCV::SLLI), SPReg)
1010 .addReg(VR)
1011 .addImm(ShiftAmount)
1013 }
1014 if (NeedProbe && RVVStackSize == 0) {
1015 // Do a probe if the align + size allocated just passed the probe size
1016 // and was not yet probed.
1017 if (SecondSPAdjustAmount < ProbeSize &&
1018 SecondSPAdjustAmount + MaxAlignment.value() >= ProbeSize) {
1019 bool IsRV64 = STI.is64Bit();
1020 BuildMI(MBB, MBBI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
1021 .addReg(RISCV::X0)
1022 .addReg(SPReg)
1023 .addImm(0)
1025 }
1026 }
1027 // FP will be used to restore the frame in the epilogue, so we need
1028 // another base register BP to record SP after re-alignment. SP will
1029 // track the current stack after allocating variable sized objects.
1030 if (hasBP(MF)) {
1031 // move BP, SP
1032 BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), BPReg)
1033 .addReg(SPReg)
1034 .addImm(0)
1036 }
1037 }
1038 }
1039}
1040
1041void RISCVFrameLowering::deallocateStack(MachineFunction &MF,
1044 const DebugLoc &DL,
1045 uint64_t &StackSize,
1046 int64_t CFAOffset) const {
1049
1050 RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize),
1052 StackSize = 0;
1053
1054 unsigned CFIIndex =
1055 MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, CFAOffset));
1056 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1057 .addCFIIndex(CFIIndex)
1059}
1060
1062 MachineBasicBlock &MBB) const {
1064 MachineFrameInfo &MFI = MF.getFrameInfo();
1065 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1067
1068 // All calls are tail calls in GHC calling conv, and functions have no
1069 // prologue/epilogue.
1071 return;
1072
1073 // Get the insert location for the epilogue. If there were no terminators in
1074 // the block, get the last instruction.
1076 DebugLoc DL;
1077 if (!MBB.empty()) {
1079 if (MBBI != MBB.end())
1080 DL = MBBI->getDebugLoc();
1081
1083
1084 // Skip to before the restores of all callee-saved registers.
1085 while (MBBI != MBB.begin() &&
1086 std::prev(MBBI)->getFlag(MachineInstr::FrameDestroy))
1087 --MBBI;
1088 }
1089
1090 const auto &CSI = MFI.getCalleeSavedInfo();
1091
1092 // Skip to before the restores of scalar callee-saved registers
1093 // FIXME: assumes exactly one instruction is used to restore each
1094 // callee-saved register.
1095 auto FirstScalarCSRRestoreInsn =
1096 std::next(MBBI, getRVVCalleeSavedInfo(MF, CSI).size());
1097
1098 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1099 uint64_t RealStackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1101 uint64_t StackSize = FirstSPAdjustAmount ? FirstSPAdjustAmount
1103 RVFI->getReservedSpillsSize();
1104 uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize();
1105 uint64_t RVVStackSize = RVFI->getRVVStackSize();
1106
1107 bool RestoreSPFromFP = RI->hasStackRealignment(MF) ||
1109 if (RVVStackSize) {
1110 // If RestoreSPFromFP the stack pointer will be restored using the frame
1111 // pointer value.
1112 if (!RestoreSPFromFP)
1113 RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, SPReg,
1114 StackOffset::getScalable(RVVStackSize),
1116
1117 if (!hasFP(MF)) {
1118 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
1119 nullptr, RI->getDwarfRegNum(SPReg, true), RealStackSize));
1120 BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
1121 TII->get(TargetOpcode::CFI_INSTRUCTION))
1122 .addCFIIndex(CFIIndex)
1124 }
1125
1126 emitCalleeSavedRVVEpilogCFI(MBB, FirstScalarCSRRestoreInsn);
1127 }
1128
1129 if (FirstSPAdjustAmount) {
1130 uint64_t SecondSPAdjustAmount =
1131 getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount;
1132 assert(SecondSPAdjustAmount > 0 &&
1133 "SecondSPAdjustAmount should be greater than zero");
1134
1135 // If RestoreSPFromFP the stack pointer will be restored using the frame
1136 // pointer value.
1137 if (!RestoreSPFromFP)
1138 RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, SPReg,
1139 StackOffset::getFixed(SecondSPAdjustAmount),
1141
1142 if (!hasFP(MF)) {
1143 unsigned CFIIndex = MF.addFrameInst(
1144 MCCFIInstruction::cfiDefCfaOffset(nullptr, FirstSPAdjustAmount));
1145 BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
1146 TII->get(TargetOpcode::CFI_INSTRUCTION))
1147 .addCFIIndex(CFIIndex)
1149 }
1150 }
1151
1152 // Restore the stack pointer using the value of the frame pointer. Only
1153 // necessary if the stack pointer was modified, meaning the stack size is
1154 // unknown.
1155 //
1156 // In order to make sure the stack point is right through the EH region,
1157 // we also need to restore stack pointer from the frame pointer if we
1158 // don't preserve stack space within prologue/epilogue for outgoing variables,
1159 // normally it's just checking the variable sized object is present or not
1160 // is enough, but we also don't preserve that at prologue/epilogue when
1161 // have vector objects in stack.
1162 if (RestoreSPFromFP) {
1163 assert(hasFP(MF) && "frame pointer should not have been eliminated");
1164 RI->adjustReg(MBB, FirstScalarCSRRestoreInsn, DL, SPReg, FPReg,
1166 getStackAlign());
1167 }
1168
1169 if (hasFP(MF)) {
1170 unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa(
1171 nullptr, RI->getDwarfRegNum(SPReg, true), RealStackSize));
1172 BuildMI(MBB, FirstScalarCSRRestoreInsn, DL,
1173 TII->get(TargetOpcode::CFI_INSTRUCTION))
1174 .addCFIIndex(CFIIndex)
1176 }
1177
1178 // Skip to after the restores of scalar callee-saved registers
1179 // FIXME: assumes exactly one instruction is used to restore each
1180 // callee-saved register.
1181 MBBI = std::next(FirstScalarCSRRestoreInsn, getUnmanagedCSI(MF, CSI).size());
1182
1183 if (getLibCallID(MF, CSI) != -1) {
1184 // tail __riscv_restore_[0-12] instruction is considered as a terminator,
1185 // therefor it is unnecessary to place any CFI instructions after it. Just
1186 // deallocate stack if needed and return.
1187 if (StackSize != 0)
1188 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1189 RVFI->getLibCallStackSize());
1190
1191 // Emit epilogue for shadow call stack.
1192 emitSCSEpilogue(MF, MBB, MBBI, DL);
1193 return;
1194 }
1195
1196 // Recover callee-saved registers.
1197 emitCFIForCSI<CFIRestoreRegisterEmitter>(MBB, MBBI, getUnmanagedCSI(MF, CSI));
1198
1199 bool ApplyPop = RVFI->isPushable(MF) && MBBI != MBB.end() &&
1200 MBBI->getOpcode() == RISCV::CM_POP;
1201 if (ApplyPop) {
1202 // Use available stack adjustment in pop instruction to deallocate stack
1203 // space. Align the stack size down to a multiple of 16. This is needed for
1204 // RVE.
1205 // FIXME: Can we increase the stack size to a multiple of 16 instead?
1206 uint64_t Spimm =
1207 std::min(alignDown(StackSize, 16), static_cast<uint64_t>(48));
1208 MBBI->getOperand(1).setImm(Spimm);
1209 StackSize -= Spimm;
1210
1211 if (StackSize != 0)
1212 deallocateStack(MF, MBB, MBBI, DL, StackSize,
1213 /*stack_adj of cm.pop instr*/ RealStackSize - StackSize);
1214
1215 auto NextI = next_nodbg(MBBI, MBB.end());
1216 if (NextI == MBB.end() || NextI->getOpcode() != RISCV::PseudoRET) {
1217 ++MBBI;
1218
1219 emitCFIForCSI<CFIRestoreRegisterEmitter>(
1221
1222 // Update CFA offset. After CM_POP SP should be equal to CFA, so CFA
1223 // offset should be a zero.
1224 unsigned CFIIndex =
1226 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
1227 .addCFIIndex(CFIIndex)
1229 }
1230 }
1231
1232 // Deallocate stack if StackSize isn't a zero yet
1233 if (StackSize != 0)
1234 deallocateStack(MF, MBB, MBBI, DL, StackSize, 0);
1235
1236 // Emit epilogue for shadow call stack.
1237 emitSCSEpilogue(MF, MBB, MBBI, DL);
1238}
1239
1242 Register &FrameReg) const {
1243 const MachineFrameInfo &MFI = MF.getFrameInfo();
1245 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1246
1247 // Callee-saved registers should be referenced relative to the stack
1248 // pointer (positive offset), otherwise use the frame pointer (negative
1249 // offset).
1250 const auto &CSI = getUnmanagedCSI(MF, MFI.getCalleeSavedInfo());
1251 int MinCSFI = 0;
1252 int MaxCSFI = -1;
1254 auto StackID = MFI.getStackID(FI);
1255
1256 assert((StackID == TargetStackID::Default ||
1257 StackID == TargetStackID::ScalableVector) &&
1258 "Unexpected stack ID for the frame object.");
1259 if (StackID == TargetStackID::Default) {
1260 assert(getOffsetOfLocalArea() == 0 && "LocalAreaOffset is not 0!");
1262 MFI.getOffsetAdjustment());
1263 } else if (StackID == TargetStackID::ScalableVector) {
1265 }
1266
1267 uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF);
1268
1269 if (CSI.size()) {
1270 MinCSFI = CSI[0].getFrameIdx();
1271 MaxCSFI = CSI[CSI.size() - 1].getFrameIdx();
1272 }
1273
1274 if (FI >= MinCSFI && FI <= MaxCSFI) {
1275 FrameReg = SPReg;
1276
1277 if (FirstSPAdjustAmount)
1278 Offset += StackOffset::getFixed(FirstSPAdjustAmount);
1279 else
1281 return Offset;
1282 }
1283
1284 if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(FI)) {
1285 // If the stack was realigned, the frame pointer is set in order to allow
1286 // SP to be restored, so we need another base register to record the stack
1287 // after realignment.
1288 // |--------------------------| -- <-- FP
1289 // | callee-allocated save | | <----|
1290 // | area for register varargs| | |
1291 // |--------------------------| | |
1292 // | callee-saved registers | | |
1293 // |--------------------------| -- |
1294 // | realignment (the size of | | |
1295 // | this area is not counted | | |
1296 // | in MFI.getStackSize()) | | |
1297 // |--------------------------| -- |-- MFI.getStackSize()
1298 // | RVV alignment padding | | |
1299 // | (not counted in | | |
1300 // | MFI.getStackSize() but | | |
1301 // | counted in | | |
1302 // | RVFI.getRVVStackSize()) | | |
1303 // |--------------------------| -- |
1304 // | RVV objects | | |
1305 // | (not counted in | | |
1306 // | MFI.getStackSize()) | | |
1307 // |--------------------------| -- |
1308 // | padding before RVV | | |
1309 // | (not counted in | | |
1310 // | MFI.getStackSize() or in | | |
1311 // | RVFI.getRVVStackSize()) | | |
1312 // |--------------------------| -- |
1313 // | scalar local variables | | <----'
1314 // |--------------------------| -- <-- BP (if var sized objects present)
1315 // | VarSize objects | |
1316 // |--------------------------| -- <-- SP
1317 if (hasBP(MF)) {
1318 FrameReg = RISCVABI::getBPReg();
1319 } else {
1320 // VarSize objects must be empty in this case!
1321 assert(!MFI.hasVarSizedObjects());
1322 FrameReg = SPReg;
1323 }
1324 } else {
1325 FrameReg = RI->getFrameRegister(MF);
1326 }
1327
1328 if (FrameReg == FPReg) {
1329 Offset += StackOffset::getFixed(RVFI->getVarArgsSaveSize());
1330 // When using FP to access scalable vector objects, we need to minus
1331 // the frame size.
1332 //
1333 // |--------------------------| -- <-- FP
1334 // | callee-allocated save | |
1335 // | area for register varargs| |
1336 // |--------------------------| |
1337 // | callee-saved registers | |
1338 // |--------------------------| | MFI.getStackSize()
1339 // | scalar local variables | |
1340 // |--------------------------| -- (Offset of RVV objects is from here.)
1341 // | RVV objects |
1342 // |--------------------------|
1343 // | VarSize objects |
1344 // |--------------------------| <-- SP
1346 assert(!RI->hasStackRealignment(MF) &&
1347 "Can't index across variable sized realign");
1348 // We don't expect any extra RVV alignment padding, as the stack size
1349 // and RVV object sections should be correct aligned in their own
1350 // right.
1352 "Inconsistent stack layout");
1354 }
1355 return Offset;
1356 }
1357
1358 // This case handles indexing off both SP and BP.
1359 // If indexing off SP, there must not be any var sized objects
1360 assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects());
1361
1362 // When using SP to access frame objects, we need to add RVV stack size.
1363 //
1364 // |--------------------------| -- <-- FP
1365 // | callee-allocated save | | <----|
1366 // | area for register varargs| | |
1367 // |--------------------------| | |
1368 // | callee-saved registers | | |
1369 // |--------------------------| -- |
1370 // | RVV alignment padding | | |
1371 // | (not counted in | | |
1372 // | MFI.getStackSize() but | | |
1373 // | counted in | | |
1374 // | RVFI.getRVVStackSize()) | | |
1375 // |--------------------------| -- |
1376 // | RVV objects | | |-- MFI.getStackSize()
1377 // | (not counted in | | |
1378 // | MFI.getStackSize()) | | |
1379 // |--------------------------| -- |
1380 // | padding before RVV | | |
1381 // | (not counted in | | |
1382 // | MFI.getStackSize()) | | |
1383 // |--------------------------| -- |
1384 // | scalar local variables | | <----'
1385 // |--------------------------| -- <-- BP (if var sized objects present)
1386 // | VarSize objects | |
1387 // |--------------------------| -- <-- SP
1388 //
1389 // The total amount of padding surrounding RVV objects is described by
1390 // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV
1391 // objects to the required alignment.
1392 if (MFI.getStackID(FI) == TargetStackID::Default) {
1393 if (MFI.isFixedObjectIndex(FI)) {
1394 assert(!RI->hasStackRealignment(MF) &&
1395 "Can't index across variable sized realign");
1397 RVFI->getRVVStackSize());
1398 } else {
1400 }
1401 } else if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
1402 // Ensure the base of the RVV stack is correctly aligned: add on the
1403 // alignment padding.
1404 int ScalarLocalVarSize = MFI.getStackSize() -
1405 RVFI->getCalleeSavedStackSize() -
1406 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
1407 Offset += StackOffset::get(ScalarLocalVarSize, RVFI->getRVVStackSize());
1408 }
1409 return Offset;
1410}
1411
1413 BitVector &SavedRegs,
1414 RegScavenger *RS) const {
1416 // Unconditionally spill RA and FP only if the function uses a frame
1417 // pointer.
1418 if (hasFP(MF)) {
1419 SavedRegs.set(RAReg);
1420 SavedRegs.set(FPReg);
1421 }
1422 // Mark BP as used if function has dedicated base pointer.
1423 if (hasBP(MF))
1424 SavedRegs.set(RISCVABI::getBPReg());
1425
1426 // When using cm.push/pop we must save X27 if we save X26.
1427 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1428 if (RVFI->isPushable(MF) && SavedRegs.test(RISCV::X26))
1429 SavedRegs.set(RISCV::X27);
1430}
1431
1432std::pair<int64_t, Align>
1433RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const {
1434 MachineFrameInfo &MFI = MF.getFrameInfo();
1435 // Create a buffer of RVV objects to allocate.
1436 SmallVector<int, 8> ObjectsToAllocate;
1437 auto pushRVVObjects = [&](int FIBegin, int FIEnd) {
1438 for (int I = FIBegin, E = FIEnd; I != E; ++I) {
1439 unsigned StackID = MFI.getStackID(I);
1440 if (StackID != TargetStackID::ScalableVector)
1441 continue;
1442 if (MFI.isDeadObjectIndex(I))
1443 continue;
1444
1445 ObjectsToAllocate.push_back(I);
1446 }
1447 };
1448 // First push RVV Callee Saved object, then push RVV stack object
1449 std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo();
1450 const auto &RVVCSI = getRVVCalleeSavedInfo(MF, CSI);
1451 if (!RVVCSI.empty())
1452 pushRVVObjects(RVVCSI[0].getFrameIdx(),
1453 RVVCSI[RVVCSI.size() - 1].getFrameIdx() + 1);
1454 pushRVVObjects(0, MFI.getObjectIndexEnd() - RVVCSI.size());
1455
1456 // The minimum alignment is 16 bytes.
1457 Align RVVStackAlign(16);
1458 const auto &ST = MF.getSubtarget<RISCVSubtarget>();
1459
1460 if (!ST.hasVInstructions()) {
1461 assert(ObjectsToAllocate.empty() &&
1462 "Can't allocate scalable-vector objects without V instructions");
1463 return std::make_pair(0, RVVStackAlign);
1464 }
1465
1466 // Allocate all RVV locals and spills
1467 int64_t Offset = 0;
1468 for (int FI : ObjectsToAllocate) {
1469 // ObjectSize in bytes.
1470 int64_t ObjectSize = MFI.getObjectSize(FI);
1471 auto ObjectAlign =
1472 std::max(Align(RISCV::RVVBitsPerBlock / 8), MFI.getObjectAlign(FI));
1473 // If the data type is the fractional vector type, reserve one vector
1474 // register for it.
1475 if (ObjectSize < (RISCV::RVVBitsPerBlock / 8))
1476 ObjectSize = (RISCV::RVVBitsPerBlock / 8);
1477 Offset = alignTo(Offset + ObjectSize, ObjectAlign);
1478 MFI.setObjectOffset(FI, -Offset);
1479 // Update the maximum alignment of the RVV stack section
1480 RVVStackAlign = std::max(RVVStackAlign, ObjectAlign);
1481 }
1482
1483 uint64_t StackSize = Offset;
1484
1485 // Ensure the alignment of the RVV stack. Since we want the most-aligned
1486 // object right at the bottom (i.e., any padding at the top of the frame),
1487 // readjust all RVV objects down by the alignment padding.
1488 // Stack size and offsets are multiples of vscale, stack alignment is in
1489 // bytes, we can divide stack alignment by minimum vscale to get a maximum
1490 // stack alignment multiple of vscale.
1491 auto VScale =
1492 std::max<uint64_t>(ST.getRealMinVLen() / RISCV::RVVBitsPerBlock, 1);
1493 if (auto RVVStackAlignVScale = RVVStackAlign.value() / VScale) {
1494 if (auto AlignmentPadding =
1495 offsetToAlignment(StackSize, Align(RVVStackAlignVScale))) {
1496 StackSize += AlignmentPadding;
1497 for (int FI : ObjectsToAllocate)
1498 MFI.setObjectOffset(FI, MFI.getObjectOffset(FI) - AlignmentPadding);
1499 }
1500 }
1501
1502 return std::make_pair(StackSize, RVVStackAlign);
1503}
1504
1506 // For RVV spill, scalable stack offsets computing requires up to two scratch
1507 // registers
1508 static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2;
1509
1510 // For RVV spill, non-scalable stack offsets computing requires up to one
1511 // scratch register.
1512 static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1;
1513
1514 // ADDI instruction's destination register can be used for computing
1515 // offsets. So Scalable stack offsets require up to one scratch register.
1516 static constexpr unsigned ScavSlotsADDIScalableObject = 1;
1517
1518 static constexpr unsigned MaxScavSlotsNumKnown =
1519 std::max({ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject,
1520 ScavSlotsNumRVVSpillNonScalableObject});
1521
1522 unsigned MaxScavSlotsNum = 0;
1524 return false;
1525 for (const MachineBasicBlock &MBB : MF)
1526 for (const MachineInstr &MI : MBB) {
1527 bool IsRVVSpill = RISCV::isRVVSpill(MI);
1528 for (auto &MO : MI.operands()) {
1529 if (!MO.isFI())
1530 continue;
1531 bool IsScalableVectorID = MF.getFrameInfo().getStackID(MO.getIndex()) ==
1533 if (IsRVVSpill) {
1534 MaxScavSlotsNum = std::max(
1535 MaxScavSlotsNum, IsScalableVectorID
1536 ? ScavSlotsNumRVVSpillScalableObject
1537 : ScavSlotsNumRVVSpillNonScalableObject);
1538 } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) {
1539 MaxScavSlotsNum =
1540 std::max(MaxScavSlotsNum, ScavSlotsADDIScalableObject);
1541 }
1542 }
1543 if (MaxScavSlotsNum == MaxScavSlotsNumKnown)
1544 return MaxScavSlotsNumKnown;
1545 }
1546 return MaxScavSlotsNum;
1547}
1548
1549static bool hasRVVFrameObject(const MachineFunction &MF) {
1550 // Originally, the function will scan all the stack objects to check whether
1551 // if there is any scalable vector object on the stack or not. However, it
1552 // causes errors in the register allocator. In issue 53016, it returns false
1553 // before RA because there is no RVV stack objects. After RA, it returns true
1554 // because there are spilling slots for RVV values during RA. It will not
1555 // reserve BP during register allocation and generate BP access in the PEI
1556 // pass due to the inconsistent behavior of the function.
1557 //
1558 // The function is changed to use hasVInstructions() as the return value. It
1559 // is not precise, but it can make the register allocation correct.
1560 //
1561 // FIXME: Find a better way to make the decision or revisit the solution in
1562 // D103622.
1563 //
1564 // Refer to https://github.com/llvm/llvm-project/issues/53016.
1565 return MF.getSubtarget<RISCVSubtarget>().hasVInstructions();
1566}
1567
1569 const RISCVInstrInfo &TII) {
1570 unsigned FnSize = 0;
1571 for (auto &MBB : MF) {
1572 for (auto &MI : MBB) {
1573 // Far branches over 20-bit offset will be relaxed in branch relaxation
1574 // pass. In the worst case, conditional branches will be relaxed into
1575 // the following instruction sequence. Unconditional branches are
1576 // relaxed in the same way, with the exception that there is no first
1577 // branch instruction.
1578 //
1579 // foo
1580 // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes
1581 // sd s11, 0(sp) # 4 bytes, or 2 bytes in RVC
1582 // jump .restore, s11 # 8 bytes
1583 // .rev_cond
1584 // bar
1585 // j .dest_bb # 4 bytes, or 2 bytes in RVC
1586 // .restore:
1587 // ld s11, 0(sp) # 4 bytes, or 2 bytes in RVC
1588 // .dest:
1589 // baz
1590 if (MI.isConditionalBranch())
1591 FnSize += TII.getInstSizeInBytes(MI);
1592 if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) {
1594 FnSize += 2 + 8 + 2 + 2;
1595 else
1596 FnSize += 4 + 8 + 4 + 4;
1597 continue;
1598 }
1599
1600 FnSize += TII.getInstSizeInBytes(MI);
1601 }
1602 }
1603 return FnSize;
1604}
1605
1607 MachineFunction &MF, RegScavenger *RS) const {
1608 const RISCVRegisterInfo *RegInfo =
1609 MF.getSubtarget<RISCVSubtarget>().getRegisterInfo();
1610 const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
1611 MachineFrameInfo &MFI = MF.getFrameInfo();
1612 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1613 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1614
1615 int64_t RVVStackSize;
1616 Align RVVStackAlign;
1617 std::tie(RVVStackSize, RVVStackAlign) = assignRVVStackObjectOffsets(MF);
1618
1619 RVFI->setRVVStackSize(RVVStackSize);
1620 RVFI->setRVVStackAlign(RVVStackAlign);
1621
1622 if (hasRVVFrameObject(MF)) {
1623 // Ensure the entire stack is aligned to at least the RVV requirement: some
1624 // scalable-vector object alignments are not considered by the
1625 // target-independent code.
1626 MFI.ensureMaxAlignment(RVVStackAlign);
1627 }
1628
1629 unsigned ScavSlotsNum = 0;
1630
1631 // estimateStackSize has been observed to under-estimate the final stack
1632 // size, so give ourselves wiggle-room by checking for stack size
1633 // representable an 11-bit signed field rather than 12-bits.
1634 if (!isInt<11>(MFI.estimateStackSize(MF)))
1635 ScavSlotsNum = 1;
1636
1637 // Far branches over 20-bit offset require a spill slot for scratch register.
1638 bool IsLargeFunction = !isInt<20>(estimateFunctionSizeInBytes(MF, *TII));
1639 if (IsLargeFunction)
1640 ScavSlotsNum = std::max(ScavSlotsNum, 1u);
1641
1642 // RVV loads & stores have no capacity to hold the immediate address offsets
1643 // so we must always reserve an emergency spill slot if the MachineFunction
1644 // contains any RVV spills.
1645 ScavSlotsNum = std::max(ScavSlotsNum, getScavSlotsNumForRVV(MF));
1646
1647 for (unsigned I = 0; I < ScavSlotsNum; I++) {
1648 int FI = MFI.CreateSpillStackObject(RegInfo->getSpillSize(*RC),
1649 RegInfo->getSpillAlign(*RC));
1651
1652 if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1)
1653 RVFI->setBranchRelaxationScratchFrameIndex(FI);
1654 }
1655
1656 unsigned Size = RVFI->getReservedSpillsSize();
1657 for (const auto &Info : MFI.getCalleeSavedInfo()) {
1658 int FrameIdx = Info.getFrameIdx();
1659 if (FrameIdx < 0 || MFI.getStackID(FrameIdx) != TargetStackID::Default)
1660 continue;
1661
1662 Size += MFI.getObjectSize(FrameIdx);
1663 }
1664 RVFI->setCalleeSavedStackSize(Size);
1665}
1666
1667// Not preserve stack space within prologue for outgoing variables when the
1668// function contains variable size objects or there are vector objects accessed
1669// by the frame pointer.
1670// Let eliminateCallFramePseudoInstr preserve stack space for it.
1672 return !MF.getFrameInfo().hasVarSizedObjects() &&
1673 !(hasFP(MF) && hasRVVFrameObject(MF));
1674}
1675
1676// Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions.
1680 DebugLoc DL = MI->getDebugLoc();
1681
1682 if (!hasReservedCallFrame(MF)) {
1683 // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and
1684 // ADJCALLSTACKUP must be converted to instructions manipulating the stack
1685 // pointer. This is necessary when there is a variable length stack
1686 // allocation (e.g. alloca), which means it's not possible to allocate
1687 // space for outgoing arguments from within the function prologue.
1688 int64_t Amount = MI->getOperand(0).getImm();
1689
1690 if (Amount != 0) {
1691 // Ensure the stack remains aligned after adjustment.
1692 Amount = alignSPAdjust(Amount);
1693
1694 if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN)
1695 Amount = -Amount;
1696
1697 const RISCVRegisterInfo &RI = *STI.getRegisterInfo();
1700 }
1701 }
1702
1703 return MBB.erase(MI);
1704}
1705
1706// We would like to split the SP adjustment to reduce prologue/epilogue
1707// as following instructions. In this way, the offset of the callee saved
1708// register could fit in a single store. Supposed that the first sp adjust
1709// amount is 2032.
1710// add sp,sp,-2032
1711// sw ra,2028(sp)
1712// sw s0,2024(sp)
1713// sw s1,2020(sp)
1714// sw s3,2012(sp)
1715// sw s4,2008(sp)
1716// add sp,sp,-64
1719 const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1720 const MachineFrameInfo &MFI = MF.getFrameInfo();
1721 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
1722 uint64_t StackSize = getStackSizeWithRVVPadding(MF);
1723
1724 // Disable SplitSPAdjust if save-restore libcall is used. The callee-saved
1725 // registers will be pushed by the save-restore libcalls, so we don't have to
1726 // split the SP adjustment in this case.
1727 if (RVFI->getReservedSpillsSize())
1728 return 0;
1729
1730 // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed
1731 // 12-bit and there exists a callee-saved register needing to be pushed.
1732 if (!isInt<12>(StackSize) && (CSI.size() > 0)) {
1733 // FirstSPAdjustAmount is chosen at most as (2048 - StackAlign) because
1734 // 2048 will cause sp = sp + 2048 in the epilogue to be split into multiple
1735 // instructions. Offsets smaller than 2048 can fit in a single load/store
1736 // instruction, and we have to stick with the stack alignment. 2048 has
1737 // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for
1738 // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment.
1739 const uint64_t StackAlign = getStackAlign().value();
1740
1741 // Amount of (2048 - StackAlign) will prevent callee saved and restored
1742 // instructions be compressed, so try to adjust the amount to the largest
1743 // offset that stack compression instructions accept when target supports
1744 // compression instructions.
1745 if (STI.hasStdExtCOrZca()) {
1746 // The compression extensions may support the following instructions:
1747 // riscv32: c.lwsp rd, offset[7:2] => 2^(6 + 2)
1748 // c.swsp rs2, offset[7:2] => 2^(6 + 2)
1749 // c.flwsp rd, offset[7:2] => 2^(6 + 2)
1750 // c.fswsp rs2, offset[7:2] => 2^(6 + 2)
1751 // riscv64: c.ldsp rd, offset[8:3] => 2^(6 + 3)
1752 // c.sdsp rs2, offset[8:3] => 2^(6 + 3)
1753 // c.fldsp rd, offset[8:3] => 2^(6 + 3)
1754 // c.fsdsp rs2, offset[8:3] => 2^(6 + 3)
1755 const uint64_t RVCompressLen = STI.getXLen() * 8;
1756 // Compared with amount (2048 - StackAlign), StackSize needs to
1757 // satisfy the following conditions to avoid using more instructions
1758 // to adjust the sp after adjusting the amount, such as
1759 // StackSize meets the condition (StackSize <= 2048 + RVCompressLen),
1760 // case1: Amount is 2048 - StackAlign: use addi + addi to adjust sp.
1761 // case2: Amount is RVCompressLen: use addi + addi to adjust sp.
1762 auto CanCompress = [&](uint64_t CompressLen) -> bool {
1763 if (StackSize <= 2047 + CompressLen ||
1764 (StackSize > 2048 * 2 - StackAlign &&
1765 StackSize <= 2047 * 2 + CompressLen) ||
1766 StackSize > 2048 * 3 - StackAlign)
1767 return true;
1768
1769 return false;
1770 };
1771 // In the epilogue, addi sp, sp, 496 is used to recover the sp and it
1772 // can be compressed(C.ADDI16SP, offset can be [-512, 496]), but
1773 // addi sp, sp, 512 can not be compressed. So try to use 496 first.
1774 const uint64_t ADDI16SPCompressLen = 496;
1775 if (STI.is64Bit() && CanCompress(ADDI16SPCompressLen))
1776 return ADDI16SPCompressLen;
1777 if (CanCompress(RVCompressLen))
1778 return RVCompressLen;
1779 }
1780 return 2048 - StackAlign;
1781 }
1782 return 0;
1783}
1784
1787 std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
1788 unsigned &MaxCSFrameIndex) const {
1789 // Early exit if no callee saved registers are modified!
1790 if (CSI.empty())
1791 return true;
1792
1793 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1794
1795 if (RVFI->isPushable(MF)) {
1796 // Determine how many GPRs we need to push and save it to RVFI.
1797 Register MaxReg = getMaxPushPopReg(MF, CSI);
1798 if (MaxReg != RISCV::NoRegister) {
1799 auto [RegEnc, PushedRegNum] = getPushPopEncodingAndNum(MaxReg);
1800 RVFI->setRVPushRegs(PushedRegNum);
1801 RVFI->setRVPushStackSize(alignTo((STI.getXLen() / 8) * PushedRegNum, 16));
1802
1803 // Use encoded number to represent registers to spill.
1804 RVFI->setRVPushRlist(RegEnc);
1805 }
1806 }
1807
1808 MachineFrameInfo &MFI = MF.getFrameInfo();
1809 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
1810
1811 for (auto &CS : CSI) {
1812 unsigned Reg = CS.getReg();
1813 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
1814 unsigned Size = RegInfo->getSpillSize(*RC);
1815
1816 // This might need a fixed stack slot.
1817 if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
1818 const auto *FII = llvm::find_if(
1819 FixedCSRFIMap, [&](auto P) { return P.first == CS.getReg(); });
1820 if (FII != std::end(FixedCSRFIMap)) {
1821 int64_t Offset;
1822 if (RVFI->isPushable(MF))
1823 Offset = -((FII->second + RVFI->getRVPushRegs() + 1) * (int64_t)Size);
1824 else
1825 Offset = FII->second * (int64_t)Size;
1826
1827 int FrameIdx = MFI.CreateFixedSpillStackObject(Size, Offset);
1828 assert(FrameIdx < 0);
1829 CS.setFrameIdx(FrameIdx);
1830 continue;
1831 }
1832 }
1833
1834 // Not a fixed slot.
1835 Align Alignment = RegInfo->getSpillAlign(*RC);
1836 // We may not be able to satisfy the desired alignment specification of
1837 // the TargetRegisterClass if the stack alignment is smaller. Use the
1838 // min.
1839 Alignment = std::min(Alignment, getStackAlign());
1840 int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
1841 if ((unsigned)FrameIdx < MinCSFrameIndex)
1842 MinCSFrameIndex = FrameIdx;
1843 if ((unsigned)FrameIdx > MaxCSFrameIndex)
1844 MaxCSFrameIndex = FrameIdx;
1845 CS.setFrameIdx(FrameIdx);
1848 }
1849
1850 // Allocate a fixed object that covers the full push or libcall size.
1851 if (RVFI->isPushable(MF)) {
1852 if (int64_t PushSize = RVFI->getRVPushStackSize())
1853 MFI.CreateFixedSpillStackObject(PushSize, -PushSize);
1854 } else if (int LibCallRegs = getLibCallID(MF, CSI) + 1) {
1855 int64_t LibCallFrameSize =
1856 alignTo((STI.getXLen() / 8) * LibCallRegs, getStackAlign());
1857 MFI.CreateFixedSpillStackObject(LibCallFrameSize, -LibCallFrameSize);
1858 }
1859
1860 return true;
1861}
1862
1866 if (CSI.empty())
1867 return true;
1868
1870 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
1871 DebugLoc DL;
1872 if (MI != MBB.end() && !MI->isDebugInstr())
1873 DL = MI->getDebugLoc();
1874
1875 // Emit CM.PUSH with base SPimm & evaluate Push stack
1877 if (RVFI->isPushable(*MF)) {
1878 unsigned PushedRegNum = RVFI->getRVPushRegs();
1879 if (PushedRegNum > 0) {
1880 // Use encoded number to represent registers to spill.
1881 int RegEnc = RVFI->getRVPushRlist();
1882 MachineInstrBuilder PushBuilder =
1883 BuildMI(MBB, MI, DL, TII.get(RISCV::CM_PUSH))
1885 PushBuilder.addImm((int64_t)RegEnc);
1886 PushBuilder.addImm(0);
1887
1888 for (unsigned i = 0; i < PushedRegNum; i++)
1889 PushBuilder.addUse(FixedCSRFIMap[i].first, RegState::Implicit);
1890 }
1891 } else if (const char *SpillLibCall = getSpillLibCallName(*MF, CSI)) {
1892 // Add spill libcall via non-callee-saved register t0.
1893 BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoCALLReg), RISCV::X5)
1894 .addExternalSymbol(SpillLibCall, RISCVII::MO_CALL)
1896
1897 // Add registers spilled in libcall as liveins.
1898 for (auto &CS : CSI)
1899 MBB.addLiveIn(CS.getReg());
1900 }
1901
1902 // Manually spill values not spilled by libcall & Push/Pop.
1903 const auto &UnmanagedCSI = getUnmanagedCSI(*MF, CSI);
1904 const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, CSI);
1905
1906 auto storeRegsToStackSlots = [&](decltype(UnmanagedCSI) CSInfo) {
1907 for (auto &CS : CSInfo) {
1908 // Insert the spill to the stack frame.
1909 Register Reg = CS.getReg();
1910 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
1912 CS.getFrameIdx(), RC, TRI, Register(),
1914 }
1915 };
1916 storeRegsToStackSlots(UnmanagedCSI);
1917 storeRegsToStackSlots(RVVCSI);
1918
1919 return true;
1920}
1921
1922static unsigned getCalleeSavedRVVNumRegs(const Register &BaseReg) {
1923 return RISCV::VRRegClass.contains(BaseReg) ? 1
1924 : RISCV::VRM2RegClass.contains(BaseReg) ? 2
1925 : RISCV::VRM4RegClass.contains(BaseReg) ? 4
1926 : 8;
1927}
1928
1930 const Register &Reg) {
1931 MCRegister BaseReg = TRI.getSubReg(Reg, RISCV::sub_vrm1_0);
1932 // If it's not a grouped vector register, it doesn't have subregister, so
1933 // the base register is just itself.
1934 if (BaseReg == RISCV::NoRegister)
1935 BaseReg = Reg;
1936 return BaseReg;
1937}
1938
1939void RISCVFrameLowering::emitCalleeSavedRVVPrologCFI(
1942 const MachineFrameInfo &MFI = MF->getFrameInfo();
1944 const TargetInstrInfo &TII = *STI.getInstrInfo();
1947
1948 const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, MFI.getCalleeSavedInfo());
1949 if (RVVCSI.empty())
1950 return;
1951
1952 uint64_t FixedSize = getStackSizeWithRVVPadding(*MF);
1953 if (!HasFP) {
1954 uint64_t ScalarLocalVarSize =
1955 MFI.getStackSize() - RVFI->getCalleeSavedStackSize() -
1956 RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding();
1957 FixedSize -= ScalarLocalVarSize;
1958 }
1959
1960 for (auto &CS : RVVCSI) {
1961 // Insert the spill to the stack frame.
1962 int FI = CS.getFrameIdx();
1963 MCRegister BaseReg = getRVVBaseRegister(TRI, CS.getReg());
1964 unsigned NumRegs = getCalleeSavedRVVNumRegs(CS.getReg());
1965 for (unsigned i = 0; i < NumRegs; ++i) {
1966 unsigned CFIIndex = MF->addFrameInst(createDefCFAOffset(
1967 TRI, BaseReg + i, -FixedSize, MFI.getObjectOffset(FI) / 8 + i));
1968 BuildMI(MBB, MI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
1969 .addCFIIndex(CFIIndex)
1971 }
1972 }
1973}
1974
1975void RISCVFrameLowering::emitCalleeSavedRVVEpilogCFI(
1978 const MachineFrameInfo &MFI = MF->getFrameInfo();
1980 const TargetInstrInfo &TII = *STI.getInstrInfo();
1983
1984 const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, MFI.getCalleeSavedInfo());
1985 for (auto &CS : RVVCSI) {
1986 MCRegister BaseReg = getRVVBaseRegister(TRI, CS.getReg());
1987 unsigned NumRegs = getCalleeSavedRVVNumRegs(CS.getReg());
1988 for (unsigned i = 0; i < NumRegs; ++i) {
1989 unsigned CFIIndex = MF->addFrameInst(MCCFIInstruction::createRestore(
1990 nullptr, RI->getDwarfRegNum(BaseReg + i, true)));
1991 BuildMI(MBB, MI, DL, TII.get(TargetOpcode::CFI_INSTRUCTION))
1992 .addCFIIndex(CFIIndex)
1994 }
1995 }
1996}
1997
2001 if (CSI.empty())
2002 return true;
2003
2005 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
2006 DebugLoc DL;
2007 if (MI != MBB.end() && !MI->isDebugInstr())
2008 DL = MI->getDebugLoc();
2009
2010 // Manually restore values not restored by libcall & Push/Pop.
2011 // Reverse the restore order in epilog. In addition, the return
2012 // address will be restored first in the epilogue. It increases
2013 // the opportunity to avoid the load-to-use data hazard between
2014 // loading RA and return by RA. loadRegFromStackSlot can insert
2015 // multiple instructions.
2016 const auto &UnmanagedCSI = getUnmanagedCSI(*MF, CSI);
2017 const auto &RVVCSI = getRVVCalleeSavedInfo(*MF, CSI);
2018
2019 auto loadRegFromStackSlot = [&](decltype(UnmanagedCSI) CSInfo) {
2020 for (auto &CS : CSInfo) {
2021 Register Reg = CS.getReg();
2022 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
2023 TII.loadRegFromStackSlot(MBB, MI, Reg, CS.getFrameIdx(), RC, TRI,
2025 assert(MI != MBB.begin() &&
2026 "loadRegFromStackSlot didn't insert any code!");
2027 }
2028 };
2029 loadRegFromStackSlot(RVVCSI);
2030 loadRegFromStackSlot(UnmanagedCSI);
2031
2033 if (RVFI->isPushable(*MF)) {
2034 int RegEnc = RVFI->getRVPushRlist();
2036 MachineInstrBuilder PopBuilder =
2037 BuildMI(MBB, MI, DL, TII.get(RISCV::CM_POP))
2039 // Use encoded number to represent registers to restore.
2040 PopBuilder.addImm(RegEnc);
2041 PopBuilder.addImm(0);
2042
2043 for (unsigned i = 0; i < RVFI->getRVPushRegs(); i++)
2044 PopBuilder.addDef(FixedCSRFIMap[i].first, RegState::ImplicitDefine);
2045 }
2046 } else {
2047 const char *RestoreLibCall = getRestoreLibCallName(*MF, CSI);
2048 if (RestoreLibCall) {
2049 // Add restore libcall via tail call.
2051 BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoTAIL))
2052 .addExternalSymbol(RestoreLibCall, RISCVII::MO_CALL)
2054
2055 // Remove trailing returns, since the terminator is now a tail call to the
2056 // restore function.
2057 if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) {
2058 NewMI->copyImplicitOps(*MF, *MI);
2059 MI->eraseFromParent();
2060 }
2061 }
2062 }
2063 return true;
2064}
2065
2067 // Keep the conventional code flow when not optimizing.
2068 if (MF.getFunction().hasOptNone())
2069 return false;
2070
2071 return true;
2072}
2073
2075 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2076 const MachineFunction *MF = MBB.getParent();
2077 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2078
2079 if (!RVFI->useSaveRestoreLibCalls(*MF))
2080 return true;
2081
2082 // Inserting a call to a __riscv_save libcall requires the use of the register
2083 // t0 (X5) to hold the return address. Therefore if this register is already
2084 // used we can't insert the call.
2085
2086 RegScavenger RS;
2087 RS.enterBasicBlock(*TmpMBB);
2088 return !RS.isRegUsed(RISCV::X5);
2089}
2090
2092 const MachineFunction *MF = MBB.getParent();
2093 MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
2094 const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
2095
2096 if (!RVFI->useSaveRestoreLibCalls(*MF))
2097 return true;
2098
2099 // Using the __riscv_restore libcalls to restore CSRs requires a tail call.
2100 // This means if we still need to continue executing code within this function
2101 // the restore cannot take place in this basic block.
2102
2103 if (MBB.succ_size() > 1)
2104 return false;
2105
2106 MachineBasicBlock *SuccMBB =
2107 MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin();
2108
2109 // Doing a tail call should be safe if there are no successors, because either
2110 // we have a returning block or the end of the block is unreachable, so the
2111 // restore will be eliminated regardless.
2112 if (!SuccMBB)
2113 return true;
2114
2115 // The successor can only contain a return, since we would effectively be
2116 // replacing the successor with our own tail return at the end of our block.
2117 return SuccMBB->isReturnBlock() && SuccMBB->size() == 1;
2118}
2119
2121 switch (ID) {
2124 return true;
2128 return false;
2129 }
2130 llvm_unreachable("Invalid TargetStackID::Value");
2131}
2132
2135}
2136
2137// Synthesize the probe loop.
2140 Register TargetReg, bool IsRVV) {
2141 assert(TargetReg != RISCV::X2 && "New top of stack cannot already be in SP");
2142
2143 auto &Subtarget = MF.getSubtarget<RISCVSubtarget>();
2144 const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
2145 bool IsRV64 = Subtarget.is64Bit();
2146 Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
2147 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
2148 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
2149
2150 MachineFunction::iterator MBBInsertPoint = std::next(MBB.getIterator());
2151 MachineBasicBlock *LoopTestMBB =
2153 MF.insert(MBBInsertPoint, LoopTestMBB);
2155 MF.insert(MBBInsertPoint, ExitMBB);
2157 Register ScratchReg = RISCV::X7;
2158
2159 // ScratchReg = ProbeSize
2160 TII->movImm(MBB, MBBI, DL, ScratchReg, ProbeSize, Flags);
2161
2162 // LoopTest:
2163 // SUB SP, SP, ProbeSize
2164 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::SUB), SPReg)
2165 .addReg(SPReg)
2166 .addReg(ScratchReg)
2167 .setMIFlags(Flags);
2168
2169 // s[d|w] zero, 0(sp)
2170 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL,
2171 TII->get(IsRV64 ? RISCV::SD : RISCV::SW))
2172 .addReg(RISCV::X0)
2173 .addReg(SPReg)
2174 .addImm(0)
2175 .setMIFlags(Flags);
2176
2177 if (IsRVV) {
2178 // SUB TargetReg, TargetReg, ProbeSize
2179 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::SUB),
2180 TargetReg)
2181 .addReg(TargetReg)
2182 .addReg(ScratchReg)
2183 .setMIFlags(Flags);
2184
2185 // BGE TargetReg, ProbeSize, LoopTest
2186 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::BGE))
2187 .addReg(TargetReg)
2188 .addReg(ScratchReg)
2189 .addMBB(LoopTestMBB)
2190 .setMIFlags(Flags);
2191
2192 } else {
2193 // BNE SP, TargetReg, LoopTest
2194 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(RISCV::BNE))
2195 .addReg(SPReg)
2196 .addReg(TargetReg)
2197 .addMBB(LoopTestMBB)
2198 .setMIFlags(Flags);
2199 }
2200
2201 ExitMBB->splice(ExitMBB->end(), &MBB, std::next(MBBI), MBB.end());
2203
2204 LoopTestMBB->addSuccessor(ExitMBB);
2205 LoopTestMBB->addSuccessor(LoopTestMBB);
2206 MBB.addSuccessor(LoopTestMBB);
2207 // Update liveins.
2208 fullyRecomputeLiveIns({ExitMBB, LoopTestMBB});
2209}
2210
2211void RISCVFrameLowering::inlineStackProbe(MachineFunction &MF,
2212 MachineBasicBlock &MBB) const {
2213 // Get the instructions that need to be replaced. We emit at most two of
2214 // these. Remember them in order to avoid complications coming from the need
2215 // to traverse the block while potentially creating more blocks.
2217 for (MachineInstr &MI : MBB) {
2218 unsigned Opc = MI.getOpcode();
2219 if (Opc == RISCV::PROBED_STACKALLOC ||
2220 Opc == RISCV::PROBED_STACKALLOC_RVV) {
2221 ToReplace.push_back(&MI);
2222 }
2223 }
2224
2225 for (MachineInstr *MI : ToReplace) {
2226 if (MI->getOpcode() == RISCV::PROBED_STACKALLOC ||
2227 MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV) {
2228 MachineBasicBlock::iterator MBBI = MI->getIterator();
2230 Register TargetReg = MI->getOperand(1).getReg();
2231 emitStackProbeInline(MF, MBB, MBBI, DL, TargetReg,
2232 (MI->getOpcode() == RISCV::PROBED_STACKALLOC_RVV));
2234 }
2235 }
2236}
static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
dxil DXContainer Global Emitter
This file contains constants used for implementing Dwarf debug support.
uint64_t Size
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
static uint64_t estimateFunctionSizeInBytes(const LoongArchInstrInfo *TII, const MachineFunction &MF)
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
#define P(N)
static constexpr Register SPReg
static constexpr Register FPReg
static MCRegister getRVVBaseRegister(const RISCVRegisterInfo &TRI, const Register &Reg)
static void emitStackProbeInline(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL, Register TargetReg, bool IsRVV)
static const char * getRestoreLibCallName(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static const char * getSpillLibCallName(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static bool hasRVVFrameObject(const MachineFunction &MF)
static std::pair< unsigned, unsigned > getPushPopEncodingAndNum(const Register MaxReg)
static const std::pair< MCPhysReg, int8_t > FixedCSRFIMap[]
static SmallVector< CalleeSavedInfo, 8 > getRVVCalleeSavedInfo(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static void appendScalableVectorExpression(const TargetRegisterInfo &TRI, SmallVectorImpl< char > &Expr, int FixedOffset, int ScalableOffset, llvm::raw_string_ostream &Comment)
static unsigned getCalleeSavedRVVNumRegs(const Register &BaseReg)
static Align getABIStackAlignment(RISCVABI::ABI ABI)
static SmallVector< CalleeSavedInfo, 8 > getPushOrLibCallsSavedInfo(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static int getLibCallID(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL)
static constexpr Register RAReg
static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL)
static SmallVector< CalleeSavedInfo, 8 > getUnmanagedCSI(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static Register getMaxPushPopReg(const MachineFunction &MF, const std::vector< CalleeSavedInfo > &CSI)
static unsigned getScavSlotsNumForRVV(MachineFunction &MF)
static MCCFIInstruction createDefCFAOffset(const TargetRegisterInfo &TRI, Register Reg, uint64_t FixedOffset, uint64_t ScalableOffset)
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:163
bool test(unsigned Idx) const
Definition: BitVector.h:461
BitVector & set()
Definition: BitVector.h:351
The CalleeSavedInfo class tracks the information need to locate where a callee saved register is in t...
Register getReg() const
A debug info location.
Definition: DebugLoc.h:33
Diagnostic information for unsupported feature in backend.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:277
bool hasOptNone() const
Do not optimize this function (-O0).
Definition: Function.h:701
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:369
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:731
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
Load the specified register of the given register class from the specified stack frame index.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
static MCCFIInstruction createDefCfaRegister(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_def_cfa_register modifies a rule for computing CFA.
Definition: MCDwarf.h:582
static MCCFIInstruction createRestore(MCSymbol *L, unsigned Register, SMLoc Loc={})
.cfi_restore says that the rule for Register is now the same as it was at the beginning of the functi...
Definition: MCDwarf.h:656
static MCCFIInstruction cfiDefCfa(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa defines a rule for computing CFA as: take address from Register and add Offset to it.
Definition: MCDwarf.h:575
static MCCFIInstruction createOffset(MCSymbol *L, unsigned Register, int64_t Offset, SMLoc Loc={})
.cfi_offset Previous value of Register is saved at offset Offset from CFA.
Definition: MCDwarf.h:617
static MCCFIInstruction cfiDefCfaOffset(MCSymbol *L, int64_t Offset, SMLoc Loc={})
.cfi_def_cfa_offset modifies a rule for computing CFA.
Definition: MCDwarf.h:590
static MCCFIInstruction createEscape(MCSymbol *L, StringRef Vals, SMLoc Loc={}, StringRef Comment="")
.cfi_escape Allows the user to add arbitrary bytes to the unwind info.
Definition: MCDwarf.h:687
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:33
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned succ_size() const
bool isReturnBlock() const
Convenience function that returns true if the block ends in a return instruction.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasVarSizedObjects() const
This method may be called any time after instruction selection is complete to determine if the stack ...
uint64_t getStackSize() const
Return the number of bytes that must be allocated to hold all of the fixed size frame objects.
bool adjustsStack() const
Return true if this function adjusts the stack – e.g., when calling another function.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void ensureMaxAlignment(Align Alignment)
Make sure the function is at least Align bytes aligned.
bool isFrameAddressTaken() const
This method may be called any time after instruction selection is complete to determine if there is a...
Align getMaxAlign() const
Return the alignment in bytes that this function must be aligned to, which is greater than the defaul...
void setObjectOffset(int ObjectIdx, int64_t SPOffset)
Set the stack frame offset of the specified object.
uint64_t getMaxCallFrameSize() const
Return the maximum size of a call frame that must be allocated for an outgoing function call.
int64_t getOffsetAdjustment() const
Return the correction for frame offsets.
int CreateSpillStackObject(uint64_t Size, Align Alignment)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
uint64_t estimateStackSize(const MachineFunction &MF) const
Estimate and return the size of the stack frame.
void setStackID(int ObjectIdx, uint8_t ID)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isMaxCallFrameSizeComputed() const
const std::vector< CalleeSavedInfo > & getCalleeSavedInfo() const
Returns a reference to call saved info vector for the current function.
int getObjectIndexEnd() const
Return one past the maximum frame object index.
int CreateFixedSpillStackObject(uint64_t Size, int64_t SPOffset, bool IsImmutable=false)
Create a spill slot at a fixed location on the stack.
uint8_t getStackID(int ObjectIdx) const
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
void setStackSize(uint64_t Size)
Set the size of the stack.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
bool isDeadObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a dead object.
unsigned addFrameInst(const MCCFIInstruction &Inst)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & addCFIIndex(unsigned CFIIndex) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:71
bool isReserved(MCRegister PhysReg) const
isReserved - Returns true when PhysReg is a reserved register.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:310
void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override
emitProlog/emitEpilog - These methods insert prolog and epilog code into the function.
uint64_t getFirstSPAdjustAmount(const MachineFunction &MF) const
bool enableShrinkWrapping(const MachineFunction &MF) const override
Returns true if the target will correctly handle shrink wrapping.
bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, ArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
spillCalleeSavedRegisters - Issues instruction(s) to spill all callee saved registers and returns tru...
bool hasBP(const MachineFunction &MF) const
bool canUseAsEpilogue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a epilogue for the target.
bool assignCalleeSavedSpillSlots(MachineFunction &MF, const TargetRegisterInfo *TRI, std::vector< CalleeSavedInfo > &CSI, unsigned &MinCSFrameIndex, unsigned &MaxCSFrameIndex) const override
assignCalleeSavedSpillSlots - Allows target to override spill slot assignment logic.
bool hasFPImpl(const MachineFunction &MF) const override
void allocateStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineFunction &MF, uint64_t Offset, uint64_t RealStackSize, bool EmitCFI, bool NeedProbe, uint64_t ProbeSize, bool DynAllocation) const
bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, MutableArrayRef< CalleeSavedInfo > CSI, const TargetRegisterInfo *TRI) const override
restoreCalleeSavedRegisters - Issues instruction(s) to restore all callee saved registers and returns...
bool hasReservedCallFrame(const MachineFunction &MF) const override
hasReservedCallFrame - Under normal circumstances, when a frame pointer is not required,...
const RISCVSubtarget & STI
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
getFrameIndexReference - This method should return the base register and offset used to reference a f...
bool isSupportedStackID(TargetStackID::Value ID) const override
void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS) const override
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override
TargetStackID::Value getStackIDForScalableVectors() const override
Returns the StackID that scalable vectors should be associated with.
void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override
processFunctionBeforeFrameFinalized - This method is called immediately before the specified function...
MachineBasicBlock::iterator eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const override
This method is called during prolog/epilog code insertion to eliminate call frame setup and destroy p...
bool canUseAsPrologue(const MachineBasicBlock &MBB) const override
Check whether or not the given MBB can be used as a prologue for the target.
RISCVFrameLowering(const RISCVSubtarget &STI)
uint64_t getStackSizeWithRVVPadding(const MachineFunction &MF) const
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
bool isPushable(const MachineFunction &MF) const
bool useSaveRestoreLibCalls(const MachineFunction &MF) const
bool hasStdExtCOrZca() const
unsigned getXLen() const
bool hasVInstructions() const
bool isRegisterReservedByUser(Register i) const override
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this functions.
unsigned getStackProbeSize(const MachineFunction &MF, Align StackAlign) const
bool isRegUsed(Register Reg, bool includeReserved=true) const
Return if a specific register is currently used.
void enterBasicBlock(MachineBasicBlock &MBB)
Start tracking liveness from the begin of basic block MBB.
void addScavengingFrameIndex(int FI)
Add a scavenging frame index.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr unsigned id() const
Definition: Register.h:103
Represents a location in source code.
Definition: SMLoc.h:23
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
void append(StringRef RHS)
Append from a StringRef.
Definition: SmallString.h:68
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:254
bool empty() const
Definition: SmallVector.h:81
size_t size() const
Definition: SmallVector.h:78
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:573
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:683
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
StackOffset holds a fixed and a scalable offset in bytes.
Definition: TypeSize.h:33
int64_t getFixed() const
Returns the fixed component of the stack.
Definition: TypeSize.h:49
int64_t getScalable() const
Returns the scalable component of the stack.
Definition: TypeSize.h:52
static StackOffset get(int64_t Fixed, int64_t Scalable)
Definition: TypeSize.h:44
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
Information about stack frame layout on the target.
bool hasFP(const MachineFunction &MF) const
hasFP - Return true if the specified function should have a dedicated frame pointer register.
virtual void determineCalleeSaves(MachineFunction &MF, BitVector &SavedRegs, RegScavenger *RS=nullptr) const
This method determines which of the registers reported by TargetRegisterInfo::getCalleeSavedRegs() sh...
int getOffsetOfLocalArea() const
getOffsetOfLocalArea - This method returns the offset of the local area from the stack pointer on ent...
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
int alignSPAdjust(int SPAdj) const
alignSPAdjust - This method aligns the stack adjustment to the correct alignment.
TargetInstrInfo - Interface to description of machine instruction set.
TargetOptions Options
bool DisableFramePointerElim(const MachineFunction &MF) const
DisableFramePointerElim - This returns true if frame pointer elimination optimization should be disab...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
const TargetRegisterClass * getMinimalPhysRegClass(MCRegister Reg, MVT VT=MVT::Other) const
Returns the Register Class of a physical register of the given type, picking the most sub register cl...
Align getSpillAlign(const TargetRegisterClass &RC) const
Return the minimum required alignment in bytes for a spill slot for a register of this class.
bool hasStackRealignment(const MachineFunction &MF) const
True if stack realignment is required and still possible.
unsigned getSpillSize(const TargetRegisterClass &RC) const
Return the size in bytes of the stack slot allocated to hold a spilled copy of a register from class ...
virtual Register getFrameRegister(const MachineFunction &MF) const =0
Debug information queries.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
self_iterator getIterator()
Definition: ilist_node.h:132
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:661
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
Definition: CallingConv.h:50
MCRegister getBPReg()
MCRegister getSCSPReg()
static constexpr unsigned RVVBitsPerBlock
bool isRVVSpill(const MachineInstr &MI)
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
@ Offset
Definition: DWP.cpp:480
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1697
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition: MathExtras.h:556
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1753
uint64_t offsetToAlignment(uint64_t Value, Align Alignment)
Returns the offset to the next integer (mod 2**64) that is greater than or equal to Value and is a mu...
Definition: Alignment.h:197
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
unsigned encodeSLEB128(int64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a SLEB128 value to an output stream.
Definition: LEB128.h:23
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1766
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:208
void fullyRecomputeLiveIns(ArrayRef< MachineBasicBlock * > MBBs)
Convenience function for recomputing live-in's for a set of MBBs until the computation converges.
Definition: LivePhysRegs.h:215
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
static bool isRVVRegClass(const TargetRegisterClass *RC)
void adjustReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator II, const DebugLoc &DL, Register DestReg, Register SrcReg, StackOffset Offset, MachineInstr::MIFlag Flag, MaybeAlign RequiredAlign) const