SIISelLowering.cpp File Reference

Custom DAG lowering for SI. More...

#include "SIISelLowering.h"
#include "AMDGPU.h"
#include "AMDGPUIntrinsicInfo.h"
#include "AMDGPUSubtarget.h"
#include "AMDGPUTargetMachine.h"
#include "SIDefines.h"
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Twine.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/DAGCombine.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MachineValueType.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <cassert>
#include <cmath>
#include <cstdint>
#include <iterator>
#include <tuple>
#include <utility>
#include <vector>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "si-lower"

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")
static unsigned	findFirstFreeSGPR (CCState &CCInfo)
static MVT	memVTFromAggregate (Type *Ty)
static bool	isFlatGlobalAddrSpace (unsigned AS)
static void	processShaderInputArgs (SmallVectorImpl< ISD::InputArg > &Splits, CallingConv::ID CallConv, ArrayRef< ISD::InputArg > Ins, BitVector &Skipped, FunctionType *FType, SIMachineFunctionInfo *Info)
static void	allocateSpecialEntryInputVGPRs (CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static ArgDescriptor	allocateVGPR32Input (CCState &CCInfo)
static ArgDescriptor	allocateSGPR32InputImpl (CCState &CCInfo, const TargetRegisterClass *RC, unsigned NumArgRegs)
static ArgDescriptor	allocateSGPR32Input (CCState &CCInfo)
static ArgDescriptor	allocateSGPR64Input (CCState &CCInfo)
static void	allocateSpecialInputVGPRs (CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static void	allocateSpecialInputSGPRs (CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static void	allocateHSAUserSGPRs (CCState &CCInfo, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static void	allocateSystemSGPRs (CCState &CCInfo, MachineFunction &MF, SIMachineFunctionInfo &Info, CallingConv::ID CallConv, bool IsShader)
static void	reservePrivateMemoryRegs (const TargetMachine &TM, MachineFunction &MF, const SIRegisterInfo &TRI, SIMachineFunctionInfo &Info)
static bool	canGuaranteeTCO (CallingConv::ID CC)
static bool	mayTailCallThisCC (CallingConv::ID CC)
	Return true if we might ever do TCO for calls with this calling convention. More...
static MachineBasicBlock::iterator	emitLoadM0FromVGPRLoop (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB, const DebugLoc &DL, const MachineOperand &IdxReg, unsigned InitReg, unsigned ResultReg, unsigned PhiReg, unsigned InitSaveExecReg, int Offset, bool UseGPRIdxMode, bool IsIndirectSrc)
static MachineBasicBlock::iterator	loadM0FromVGPR (const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineInstr &MI, unsigned InitResultReg, unsigned PhiReg, int Offset, bool UseGPRIdxMode, bool IsIndirectSrc)
static std::pair< unsigned, int >	computeIndirectRegAndOffset (const SIRegisterInfo &TRI, const TargetRegisterClass *SuperRC, unsigned VecReg, int Offset)
static bool	setM0ToIndexFromSGPR (const SIInstrInfo *TII, MachineRegisterInfo &MRI, MachineInstr &MI, int Offset, bool UseGPRIdxMode, bool IsIndirectSrc)
static MachineBasicBlock *	emitIndirectSrc (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static unsigned	getMOVRELDPseudo (const SIRegisterInfo &TRI, const TargetRegisterClass *VecRC)
static MachineBasicBlock *	emitIndirectDst (MachineInstr &MI, MachineBasicBlock &MBB, const GCNSubtarget &ST)
static SDValue	adjustLoadValueTypeImpl (SDValue Result, EVT LoadVT, const SDLoc &DL, SelectionDAG &DAG, bool Unpacked)
static SDValue	lowerICMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDValue	lowerFCMPIntrinsic (const SITargetLowering &TLI, SDNode *N, SelectionDAG &DAG)
static SDNode *	findUser (SDValue Value, unsigned Opcode)
	Helper function for LowerBRCOND. More...
static SDValue	buildPCRelGlobalAddress (SelectionDAG &DAG, const GlobalValue *GV, const SDLoc &DL, unsigned Offset, EVT PtrVT, unsigned GAFlags=SIInstrInfo::MO_NONE)
static SDValue	emitNonHSAIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue	emitRemovedIntrinsicError (SelectionDAG &DAG, const SDLoc &DL, EVT VT)
static SDValue	getBuildDwordsVector (SelectionDAG &DAG, SDLoc DL, ArrayRef< SDValue > Elts)
static bool	parseCachePolicy (SDValue CachePolicy, SelectionDAG &DAG, SDValue *GLC, SDValue *SLC)
static SDValue	constructRetValue (SelectionDAG &DAG, MachineSDNode *Result, ArrayRef< EVT > ResultTypes, bool IsTexFail, bool Unpacked, bool IsD16, int DMaskPop, int NumVDataDwords, const SDLoc &DL, LLVMContext &Context)
static bool	parseTexFail (SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, SDValue *LWE, bool &IsTexFail)
static SDValue	getLoadExtOrTrunc (SelectionDAG &DAG, ISD::LoadExtType ExtType, SDValue Op, const SDLoc &SL, EVT VT)
static SDValue	getFPBinOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue GlueChain)
static SDValue	getFPTernOp (SelectionDAG &DAG, unsigned Opcode, const SDLoc &SL, EVT VT, SDValue A, SDValue B, SDValue C, SDValue GlueChain)
static bool	bitOpWithConstantIsReducible (unsigned Opc, uint32_t Val)
static bool	isBoolSGPR (SDValue V)
static uint32_t	getConstantPermuteMask (uint32_t C)
static uint32_t	getPermuteMask (SelectionDAG &DAG, SDValue V)
static bool	fp16SrcZerosHighBits (unsigned Opc)
static bool	vectorEltWillFoldAway (SDValue Op)
static unsigned	minMaxOpcToMin3Max3Opc (unsigned Opc)
static ConstantFPSDNode *	getSplatConstantFP (SDValue Op)
static bool	isClampZeroToOne (SDValue A, SDValue B)
static SDValue	getMad64_32 (SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue N0, SDValue N1, SDValue N2, bool Signed)
static unsigned	SubIdx2Lane (unsigned Idx)
	Helper function for adjustWritemask. More...
static bool	isFrameIndexOp (SDValue Op)
static SDValue	buildSMovImm32 (SelectionDAG &DAG, const SDLoc &DL, uint64_t Val)
static LLVM_ATTRIBUTE_UNUSED bool	isCopyFromRegOfInlineAsm (const SDNode *N)

Variables
static cl::opt< bool >	EnableVGPRIndexMode ("amdgpu-vgpr-index-mode", cl::desc("Use GPR indexing mode instead of movrel for vector indexing"), cl::init(false))
static cl::opt< unsigned >	AssumeFrameIndexHighZeroBits ("amdgpu-frame-index-zero-bits", cl::desc("High bits of frame index assumed to be zero"), cl::init(5), cl::ReallyHidden)

Detailed Description

Custom DAG lowering for SI.

Definition in file SIISelLowering.cpp.

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "si-lower"

Definition at line 89 of file SIISelLowering.cpp.

Function Documentation

adjustLoadValueTypeImpl()

static SDValue adjustLoadValueTypeImpl ( SDValue  Result, EVT  LoadVT, const SDLoc &  DL, SelectionDAG &  DAG, bool  Unpacked  )

static

Definition at line 3686 of file SIISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::GCNSubtarget::hasUnpackedD16VMem(), llvm::MVT::i16, llvm::MVT::i32, llvm::ISD::INTRINSIC_W_CHAIN, llvm::EVT::isVector(), llvm::SPII::Load, llvm::MVT::Other, and llvm::ISD::TRUNCATE.

Referenced by constructRetValue().

allocateHSAUserSGPRs()

static void allocateHSAUserSGPRs ( CCState &  CCInfo, MachineFunction &  MF, const SIRegisterInfo &  TRI, SIMachineFunctionInfo &  Info  )

static

Definition at line 1610 of file SIISelLowering.cpp.

References llvm::SIMachineFunctionInfo::addDispatchID(), llvm::SIMachineFunctionInfo::addDispatchPtr(), llvm::SIMachineFunctionInfo::addFlatScratchInit(), llvm::SIMachineFunctionInfo::addImplicitBufferPtr(), llvm::SIMachineFunctionInfo::addKernargSegmentPtr(), llvm::MachineFunction::addLiveIn(), llvm::SIMachineFunctionInfo::addPrivateSegmentBuffer(), llvm::SIMachineFunctionInfo::addQueuePtr(), llvm::CCState::AllocateReg(), llvm::SIMachineFunctionInfo::hasDispatchID(), llvm::SIMachineFunctionInfo::hasDispatchPtr(), llvm::SIMachineFunctionInfo::hasFlatScratchInit(), llvm::SIMachineFunctionInfo::hasImplicitBufferPtr(), llvm::SIMachineFunctionInfo::hasKernargSegmentPtr(), llvm::SIMachineFunctionInfo::hasPrivateSegmentBuffer(), and llvm::SIMachineFunctionInfo::hasQueuePtr().

Referenced by llvm::SITargetLowering::LowerFormalArguments().

allocateSGPR32Input()

static ArgDescriptor allocateSGPR32Input ( CCState &  CCInfo )

static

Definition at line 1552 of file SIISelLowering.cpp.

References allocateSGPR32InputImpl().

Referenced by allocateSpecialInputSGPRs().

allocateSGPR32InputImpl()

static ArgDescriptor allocateSGPR32InputImpl ( CCState &  CCInfo, const TargetRegisterClass *  RC, unsigned  NumArgRegs  )

static

Definition at line 1535 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), assert(), llvm::TargetRegisterClass::begin(), llvm::ArgDescriptor::createRegister(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::makeArrayRef(), Reg, llvm::report_fatal_error(), and llvm::ArrayRef< T >::size().

Referenced by allocateSGPR32Input(), and allocateSGPR64Input().

allocateSGPR64Input()

static ArgDescriptor allocateSGPR64Input ( CCState &  CCInfo )

static

Definition at line 1556 of file SIISelLowering.cpp.

References allocateSGPR32InputImpl().

Referenced by allocateSpecialInputSGPRs().

allocateSpecialEntryInputVGPRs()

static void allocateSpecialEntryInputVGPRs ( CCState &  CCInfo, MachineFunction &  MF, const SIRegisterInfo &  TRI, SIMachineFunctionInfo &  Info  )

static

Definition at line 1484 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::ArgDescriptor::createRegister(), llvm::SIMachineFunctionInfo::hasWorkItemIDX(), llvm::SIMachineFunctionInfo::hasWorkItemIDY(), llvm::SIMachineFunctionInfo::hasWorkItemIDZ(), Reg, llvm::SIMachineFunctionInfo::setWorkItemIDX(), llvm::SIMachineFunctionInfo::setWorkItemIDY(), and llvm::SIMachineFunctionInfo::setWorkItemIDZ().

Referenced by llvm::SITargetLowering::LowerFormalArguments().

allocateSpecialInputSGPRs()

static void allocateSpecialInputSGPRs ( CCState &  CCInfo, MachineFunction &  MF, const SIRegisterInfo &  TRI, SIMachineFunctionInfo &  Info  )

static

Definition at line 1574 of file SIISelLowering.cpp.

References allocateSGPR32Input(), allocateSGPR64Input(), llvm::AMDGPUFunctionArgInfo::DispatchPtr, llvm::SIMachineFunctionInfo::getArgInfo(), llvm::SIMachineFunctionInfo::hasDispatchID(), llvm::SIMachineFunctionInfo::hasDispatchPtr(), llvm::SIMachineFunctionInfo::hasImplicitArgPtr(), llvm::SIMachineFunctionInfo::hasKernargSegmentPtr(), llvm::SIMachineFunctionInfo::hasQueuePtr(), llvm::SIMachineFunctionInfo::hasWorkGroupIDX(), llvm::SIMachineFunctionInfo::hasWorkGroupIDY(), and llvm::SIMachineFunctionInfo::hasWorkGroupIDZ().

Referenced by llvm::SITargetLowering::LowerFormalArguments().

allocateSpecialInputVGPRs()

static void allocateSpecialInputVGPRs ( CCState &  CCInfo, MachineFunction &  MF, const SIRegisterInfo &  TRI, SIMachineFunctionInfo &  Info  )

static

Definition at line 1560 of file SIISelLowering.cpp.

References allocateVGPR32Input(), llvm::SIMachineFunctionInfo::hasWorkItemIDX(), llvm::SIMachineFunctionInfo::hasWorkItemIDY(), llvm::SIMachineFunctionInfo::hasWorkItemIDZ(), llvm::SIMachineFunctionInfo::setWorkItemIDX(), llvm::SIMachineFunctionInfo::setWorkItemIDY(), and llvm::SIMachineFunctionInfo::setWorkItemIDZ().

Referenced by llvm::SITargetLowering::LowerFormalArguments().

allocateSystemSGPRs()

static void allocateSystemSGPRs ( CCState &  CCInfo, MachineFunction &  MF, SIMachineFunctionInfo &  Info, CallingConv::ID  CallConv, bool  IsShader  )

static

Definition at line 1662 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::SIMachineFunctionInfo::addPrivateSegmentWaveByteOffset(), llvm::SIMachineFunctionInfo::addWorkGroupIDX(), llvm::SIMachineFunctionInfo::addWorkGroupIDY(), llvm::SIMachineFunctionInfo::addWorkGroupIDZ(), llvm::SIMachineFunctionInfo::addWorkGroupInfo(), llvm::CCState::AllocateReg(), findFirstFreeSGPR(), llvm::SIMachineFunctionInfo::getPrivateSegmentWaveByteOffsetSystemSGPR(), llvm::SIMachineFunctionInfo::hasPrivateSegmentWaveByteOffset(), llvm::SIMachineFunctionInfo::hasWorkGroupIDX(), llvm::SIMachineFunctionInfo::hasWorkGroupIDY(), llvm::SIMachineFunctionInfo::hasWorkGroupIDZ(), llvm::SIMachineFunctionInfo::hasWorkGroupInfo(), Reg, and llvm::SIMachineFunctionInfo::setPrivateSegmentWaveByteOffset().

Referenced by llvm::SITargetLowering::LowerFormalArguments().

allocateVGPR32Input()

static ArgDescriptor allocateVGPR32Input ( CCState &  CCInfo )

static

Definition at line 1515 of file SIISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), llvm::CCState::AllocateReg(), llvm::CCState::AllocateStack(), assert(), llvm::sys::path::begin(), llvm::ArgDescriptor::createRegister(), llvm::ArgDescriptor::createStack(), llvm::CCState::getFirstUnallocated(), llvm::CCState::getMachineFunction(), llvm::makeArrayRef(), Reg, and llvm::ArrayRef< T >::size().

Referenced by allocateSpecialInputVGPRs().

bitOpWithConstantIsReducible()

static bool bitOpWithConstantIsReducible ( unsigned  Opc, uint32_t  Val  )

static

Definition at line 7149 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::ConstantSDNode::getAPIntValue(), llvm::GCNSubtarget::getInstrInfo(), llvm::SITargetLowering::getSubtarget(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::Hi_32(), llvm::SIInstrInfo::isInlineConstant(), llvm::Lo_32(), llvm::ISD::OR, llvm::AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(), TII, and llvm::ISD::XOR.

buildPCRelGlobalAddress()

static SDValue buildPCRelGlobalAddress ( SelectionDAG &  DAG, const GlobalValue *  GV, const SDLoc &  DL, unsigned  Offset, EVT  PtrVT, unsigned  GAFlags = SIInstrInfo::MO_NONE  )

static

Definition at line 4486 of file SIISelLowering.cpp.

References llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, AMDGPUAS::CONSTANT_ADDRESS, llvm::PointerType::get(), llvm::DataLayout::getABITypeAlignment(), llvm::GlobalAddressSDNode::getAddressSpace(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachinePointerInfo::getGOT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::EVT::getTypeForEVT(), llvm::SDValue::getValueType(), llvm::MVT::i32, AMDGPUAS::LOCAL_ADDRESS, llvm::AMDGPUTargetLowering::LowerGlobalAddress(), llvm::SIInstrInfo::MO_GOTPCREL32, llvm::SIInstrInfo::MO_NONE, llvm::SIInstrInfo::MO_REL32, llvm::MachineMemOperand::MODereferenceable, llvm::MachineMemOperand::MOInvariant, llvm::AMDGPUISD::PC_ADD_REL_OFFSET, AMDGPUAS::PRIVATE_ADDRESS, and AMDGPUAS::REGION_ADDRESS.

buildSMovImm32()

static SDValue buildSMovImm32 ( SelectionDAG &  DAG, const SDLoc &  DL, uint64_t  Val  )

static

Definition at line 9359 of file SIISelLowering.cpp.

References llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::MVT::i32.

Referenced by llvm::SITargetLowering::buildRSRC(), and llvm::SITargetLowering::wrapAddr64Rsrc().

canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC )

static

Definition at line 2346 of file SIISelLowering.cpp.

References llvm::CallingConv::Fast.

Referenced by llvm::SITargetLowering::isEligibleForTailCallOptimization(), and mayTailCallThisCC().

computeIndirectRegAndOffset()

static std::pair<unsigned, int> computeIndirectRegAndOffset ( const SIRegisterInfo &  TRI, const TargetRegisterClass *  SuperRC, unsigned  VecReg, int  Offset  )

static

Definition at line 2997 of file SIISelLowering.cpp.

Referenced by emitIndirectDst(), and emitIndirectSrc().

constructRetValue()

static SDValue constructRetValue ( SelectionDAG &  DAG, MachineSDNode *  Result, ArrayRef< EVT >  ResultTypes, bool  IsTexFail, bool  Unpacked, bool  IsD16, int  DMaskPop, int  NumVDataDwords, const SDLoc &  DL, LLVMContext &  Context  )

static

Definition at line 4670 of file SIISelLowering.cpp.

References adjustLoadValueTypeImpl(), llvm::ISD::BITCAST, llvm::SelectionDAG::ExtractVectorElements(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::i32, llvm::EVT::isVector(), N, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::MVT::v2i32, llvm::MVT::v4f16, llvm::MVT::v4i32, and llvm::MVT::v8f16.

Referenced by parseTexFail().

emitIndirectDst()

static MachineBasicBlock* emitIndirectDst ( MachineInstr &  MI, MachineBasicBlock &  MBB, const GCNSubtarget &  ST  )

static

Definition at line 3160 of file SIISelLowering.cpp.

References llvm::ARM_AM::add, llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::MachineRegisterInfo::clearKillFlags(), computeIndirectRegAndOffset(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::RegState::Define, EnableVGPRIndexMode, llvm::MachineInstr::eraseFromParent(), llvm::MachineInstr::getDebugLoc(), llvm::MachineOperand::getImm(), llvm::GCNSubtarget::getInstrInfo(), getMOVRELDPseudo(), llvm::SIInstrInfo::getNamedOperand(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), llvm::SIInstrInfo::getRegisterInfo(), I, llvm::RegState::Implicit, llvm::RegState::ImplicitDefine, llvm::MachineOperand::isReg(), loadM0FromVGPR(), MRI, setM0ToIndexFromSGPR(), SubReg, TII, TRI, llvm::RegState::Undef, and llvm::GCNSubtarget::useVGPRIndexMode().

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

emitIndirectSrc()

static MachineBasicBlock* emitIndirectSrc ( MachineInstr &  MI, MachineBasicBlock &  MBB, const GCNSubtarget &  ST  )

static

Definition at line 3069 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), computeIndirectRegAndOffset(), llvm::MachineRegisterInfo::createVirtualRegister(), EnableVGPRIndexMode, llvm::MachineInstr::eraseFromParent(), llvm::MachineInstr::getDebugLoc(), llvm::MachineOperand::getImm(), llvm::GCNSubtarget::getInstrInfo(), llvm::SIInstrInfo::getNamedOperand(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), llvm::SIInstrInfo::getRegisterInfo(), I, llvm::RegState::Implicit, loadM0FromVGPR(), MRI, setM0ToIndexFromSGPR(), SubReg, TII, TRI, llvm::RegState::Undef, and llvm::GCNSubtarget::useVGPRIndexMode().

Referenced by llvm::SITargetLowering::EmitInstrWithCustomInserter().

emitLoadM0FromVGPRLoop()

static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop ( const SIInstrInfo *  TII, MachineRegisterInfo &  MRI, MachineBasicBlock &  OrigBB, MachineBasicBlock &  LoopBB, const DebugLoc &  DL, const MachineOperand &  IdxReg, unsigned  InitReg, unsigned  ResultReg, unsigned  PhiReg, unsigned  InitSaveExecReg, int  Offset, bool  UseGPRIdxMode, bool  IsIndirectSrc  )

static

Definition at line 2842 of file SIISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::VGPRIndexMode::DST_ENABLE, llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineOperand::getSubReg(), llvm::getUndefRegState(), I, llvm::MachineOperand::isUndef(), llvm::RegState::Kill, llvm::MachineOperand::setIsUndef(), llvm::MachineRegisterInfo::setSimpleHint(), and llvm::VGPRIndexMode::SRC0_ENABLE.

Referenced by loadM0FromVGPR().

emitNonHSAIntrinsicError()

static SDValue emitNonHSAIntrinsicError ( SelectionDAG &  DAG, const SDLoc &  DL, EVT  VT  )

static

Definition at line 4590 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), llvm::SelectionDAG::getContext(), llvm::SDLoc::getDebugLoc(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().

Referenced by parseTexFail().

emitRemovedIntrinsicError()

static SDValue emitRemovedIntrinsicError ( SelectionDAG &  DAG, const SDLoc &  DL, EVT  VT  )

static

Definition at line 4599 of file SIISelLowering.cpp.

References llvm::LLVMContext::diagnose(), llvm::SelectionDAG::getContext(), llvm::SDLoc::getDebugLoc(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getUNDEF().

Referenced by parseTexFail().

findFirstFreeSGPR()

static unsigned findFirstFreeSGPR ( CCState &  CCInfo )

static

Definition at line 104 of file SIISelLowering.cpp.

References llvm::CCState::isAllocated(), llvm_unreachable, llvm::AMDGPU::HSAMD::Kernel::CodeProps::Key::NumSGPRs, and Reg.

Referenced by allocateSystemSGPRs().

findUser()

static SDNode* findUser ( SDValue  Value, unsigned  Opcode  )

static

Helper function for LowerBRCOND.

Definition at line 3928 of file SIISelLowering.cpp.

References llvm::AfterLegalizeVectorOps, llvm::Intrinsic::amdgcn_else, llvm::Intrinsic::amdgcn_end_cf, llvm::Intrinsic::amdgcn_if, llvm::Intrinsic::amdgcn_loop, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::SmallVectorImpl< T >::append(), assert(), llvm::AMDGPUISD::BFI, llvm::AMDGPUISD::BFM, llvm::ISD::BITCAST, llvm::EVT::bitsLE(), llvm::ISD::BR, llvm::ISD::BRCOND, llvm::ISD::BUILD_VECTOR, Concat, AMDGPUAS::CONSTANT_ADDRESS, AMDGPUAS::CONSTANT_ADDRESS_32BIT, llvm::ISD::CopyToReg, llvm::MachineFrameInfo::CreateFixedObject(), llvm::AMDGPUTargetLowering::CreateLiveInRegister(), llvm::LLVMContext::diagnose(), llvm::DS_Warning, llvm::dyn_cast(), E, llvm::AMDGPUISD::ELSE, llvm::GCNSubtarget::enableIEEEBit(), llvm::AMDGPUISD::ENDPGM, llvm::TargetLowering::expandFMINNUM_FMAXNUM(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f64, AMDGPUAS::FLAT_ADDRESS, llvm::ISD::FP_EXTEND, llvm::ISD::FP_TO_FP16, llvm::ISD::FTRUNC, llvm::PointerType::get(), llvm::UndefValue::get(), llvm::PointerType::getAddressSpace(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::Function::getContext(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyToReg(), llvm::SDValue::getDebugLoc(), llvm::SDLoc::getDebugLoc(), llvm::AddrSpaceCastSDNode::getDestAddressSpace(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getInfo(), llvm::Type::getInt8Ty(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::AMDGPUTargetMachine::getNullPointerValue(), llvm::SDNode::getNumValues(), llvm::SelectionDAG::getObjectPtrOffset(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::GlobalValue::getParent(), llvm::SIMachineFunctionInfo::getQueuePtrUserSGPR(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::AddrSpaceCastSDNode::getSrcAddressSpace(), llvm::SelectionDAG::getTargetConstant(), llvm::TargetLoweringBase::getTargetMachine(), llvm::TargetMachine::getTargetTriple(), llvm::GCNSubtarget::getTrapHandlerAbi(), llvm::GlobalValue::getType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), AMDGPUAS::GLOBAL_ADDRESS, llvm::GCNSubtarget::hasApertureRegs(), llvm::AMDGPUSubtarget::hasVOP3PInsts(), llvm::MipsISD::Hi, I, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::AMDGPU::Hwreg::ID_MEM_BASES, llvm::AMDGPU::Hwreg::ID_SHIFT_, llvm::AMDGPUISD::IF, Info, llvm::ISD::INSERT_VECTOR_ELT, Intr, llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, llvm::isPowerOf2_32(), llvm::GCNSubtarget::isTrapHandlerEnabled(), llvm::SDValue::isUndef(), llvm_unreachable, llvm::MipsISD::Lo, AMDGPUAS::LOCAL_ADDRESS, llvm::Log2_32(), llvm::AMDGPUISD::LOOP, llvm::MinAlign(), llvm::MachineMemOperand::MODereferenceable, llvm::MachineMemOperand::MOInvariant, llvm::AMDGPU::Hwreg::OFFSET_SHIFT_, llvm::AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE, llvm::AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::OR, llvm::MVT::Other, AMDGPUAS::PRIVATE_ADDRESS, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::SIMachineFunctionInfo::setDebuggerWorkGroupIDStackObjectIndex(), llvm::SIMachineFunctionInfo::setDebuggerWorkItemIDStackObjectIndex(), llvm::ISD::SETNE, llvm::ISD::SHL, llvm::TargetMachine::shouldAssumeDSOLocal(), llvm::AMDGPU::shouldEmitConstantsToTextSection(), llvm::SITargetLowering::splitBinaryVectorOp(), llvm::ISD::SRL, llvm::AMDGPUISD::TRAP, llvm::GCNSubtarget::TrapHandlerAbiHsa, llvm::GCNSubtarget::TrapIDLLVMDebugTrap, llvm::GCNSubtarget::TrapIDLLVMTrap, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v2f16, llvm::MVT::v2i16, llvm::MVT::v2i32, llvm::MVT::v4f16, llvm::MVT::v4i16, llvm::SDNode::value_begin(), llvm::SDNode::value_end(), llvm::AMDGPU::Hwreg::WIDTH_M1_SHIFT_, llvm::AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE, llvm::AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE, and llvm::ISD::ZERO_EXTEND.

fp16SrcZerosHighBits()

static bool fp16SrcZerosHighBits ( unsigned  Opc )

static

Definition at line 7598 of file SIISelLowering.cpp.

References llvm::AfterLegalizeDAG, llvm::ISD::BITCAST, llvm::AMDGPUISD::CLAMP, llvm::AMDGPUISD::COS_HW, llvm::MVT::f16, llvm::MVT::f32, llvm::ISD::FABS, llvm::ISD::FADD, llvm::ISD::FCANONICALIZE, llvm::ISD::FCEIL, llvm::ISD::FCOS, llvm::ISD::FDIV, llvm::ISD::FEXP, llvm::ISD::FEXP2, llvm::ISD::FFLOOR, llvm::ISD::FLOG, llvm::ISD::FLOG10, llvm::ISD::FLOG2, llvm::ISD::FMA, llvm::ISD::FMAD, llvm::AMDGPUISD::FMAD_FTZ, llvm::AMDGPUISD::FMAX3, llvm::ISD::FMAXNUM, llvm::AMDGPUISD::FMED3, llvm::AMDGPUISD::FMIN3, llvm::ISD::FMINNUM, llvm::ISD::FMUL, llvm::ISD::FNEARBYINT, llvm::AMDGPUISD::FP16_ZEXT, llvm::ISD::FP_ROUND, llvm::ISD::FPOW, llvm::ISD::FPOWI, llvm::AMDGPUISD::FRACT, llvm::ISD::FREM, llvm::ISD::FRINT, llvm::ISD::FROUND, llvm::ISD::FSIN, llvm::ISD::FSQRT, llvm::ISD::FSUB, llvm::ISD::FTRUNC, llvm::SelectionDAG::getConstant(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::AMDGPUSubtarget::has16BitInsts(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::SDValue::isUndef(), llvm::AMDGPUISD::LDEXP, llvm::BitmaskEnumDetail::Mask(), llvm::AMDGPUTargetLowering::performRcpCombine(), llvm::AMDGPUISD::RCP, llvm::AMDGPUISD::RCP_IFLAG, llvm::AMDGPUISD::RSQ, llvm::AMDGPUISD::SIN_HW, llvm::ISD::SINT_TO_FP, and llvm::ISD::UINT_TO_FP.

getBuildDwordsVector()

static SDValue getBuildDwordsVector ( SelectionDAG &  DAG, SDLoc  DL, ArrayRef< SDValue >  Elts  )

static

Definition at line 4608 of file SIISelLowering.cpp.

References assert(), llvm::ArrayRef< T >::empty(), llvm::MVT::f32, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, llvm::MVT::v2f32, llvm::MVT::v4f32, and llvm::MVT::v8f32.

Referenced by parseTexFail().

getConstantPermuteMask()

static uint32_t getConstantPermuteMask ( uint32_t  C )

static

Definition at line 7201 of file SIISelLowering.cpp.

References C.

Referenced by getPermuteMask().

getFPBinOp()

static SDValue getFPBinOp ( SelectionDAG &  DAG, unsigned  Opcode, const SDLoc &  SL, EVT  VT, SDValue  A, SDValue  B, SDValue  GlueChain  )

static

Definition at line 6646 of file SIISelLowering.cpp.

References assert(), B, llvm::ISD::FMUL, llvm::AMDGPUISD::FMUL_W_CHAIN, llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm_unreachable, and llvm::MVT::Other.

Referenced by getFPTernOp().

getFPTernOp()

static SDValue getFPTernOp ( SelectionDAG &  DAG, unsigned  Opcode, const SDLoc &  SL, EVT  VT, SDValue  A, SDValue  B, SDValue  C, SDValue  GlueChain  )

static

Definition at line 6666 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLoweringBase::allowsMemoryAccess(), assert(), llvm::AMDGPUISD::ATOMIC_CMP_SWAP, B, llvm::ISD::BITCAST, llvm::BitsToFloat(), C, llvm::AMDGPUISD::COS_HW, llvm::AMDGPUISD::CVT_F32_UBYTE0, llvm::AMDGPUISD::DIV_FIXUP, llvm::AMDGPUISD::DIV_FMAS, llvm::AMDGPUISD::DIV_SCALE, llvm::dyn_cast(), llvm::TargetLowering::expandUnalignedStore(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::ISD::FABS, llvm::ISD::FCOS, AMDGPUAS::FLAT_ADDRESS, llvm::ISD::FMA, llvm::AMDGPUISD::FMA_W_CHAIN, llvm::ISD::FMUL, llvm::ISD::FNEG, FP_DENORM_FLUSH_IN_FLUSH_OUT, FP_DENORM_FLUSH_NONE, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::AMDGPUISD::FRACT, llvm::ISD::FSIN, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::ConstantSDNode::getAPIntValue(), llvm::MemSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SDNode::getFlags(), getFPBinOp(), llvm::GCNSubtarget::getGeneration(), llvm::APInt::getHighBitsSet(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getMachineFunction(), llvm::GCNSubtarget::getMaxPrivateElementSize(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRoot(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SITargetLowering::getSetCCResultType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::APInt::getSExtValue(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTruncStore(), llvm::EVT::getTypeForEVT(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), AMDGPUAS::GLOBAL_ADDRESS, llvm::MVT::Glue, llvm::TargetLoweringBase::AddrMode::HasBaseReg, llvm::SIMachineFunctionInfo::hasFlatScratchInit(), llvm::AMDGPUSubtarget::hasFP32Denormals(), llvm::SDNodeFlags::hasNoUnsignedWrap(), llvm::SDNode::hasOneUse(), llvm::AMDGPUSubtarget::hasTrigReducedRange(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::AMDGPU::Hwreg::ID_MODE, llvm::AtomicSDNode::isCompareAndSwap(), isFlatGlobalAddrSpace(), llvm::SITargetLowering::isLegalAddressingMode(), llvm_unreachable, AMDGPUAS::LOCAL_ADDRESS, llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::AMDGPU::Hwreg::OFFSET_SHIFT_, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::TargetMachine::Options, llvm::ISD::OR, llvm::MVT::Other, AMDGPUAS::PRIVATE_ADDRESS, r0(), r1(), r2(), r3(), llvm::AMDGPUISD::RCP, llvm::TargetLowering::scalarizeVectorStore(), llvm::ISD::SELECT, llvm::ISD::SETEQ, llvm::SDNodeFlags::setNoUnsignedWrap(), llvm::ISD::SETOGT, llvm::AMDGPUISD::SETREG, llvm::SelectionDAG::setRoot(), llvm::ISD::SHL, llvm::AMDGPUISD::SIN_HW, llvm::AMDGPUSubtarget::SOUTHERN_ISLANDS, llvm::AMDGPUTargetLowering::SplitVectorStore(), llvm::SPII::Store, llvm::ISD::STORE, llvm::ISD::TokenFactor, llvm::TargetOptions::UnsafeFPMath, llvm::SelectionDAG::UpdateNodeOperands(), llvm::GCNSubtarget::useDS128(), llvm::MVT::v2i32, llvm::AMDGPU::Hwreg::WIDTH_M1_SHIFT_, X, llvm::ISD::XOR, and Y.

getLoadExtOrTrunc()

static SDValue getLoadExtOrTrunc ( SelectionDAG &  DAG, ISD::LoadExtType  ExtType, SDValue  Op, const SDLoc &  SL, EVT  VT  )

static

Definition at line 6347 of file SIISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, llvm::EVT::bitsLT(), llvm::EVT::changeTypeToInteger(), AMDGPUAS::CONSTANT_ADDRESS, AMDGPUAS::CONSTANT_ADDRESS_32BIT, llvm::TargetLowering::expandUnalignedLoad(), llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f32, AMDGPUAS::FLAT_ADDRESS, llvm::ISD::FMUL, llvm::ISD::FNEG, llvm::ISD::FSQRT, llvm::MemSDNode::getAAInfo(), llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::LoadSDNode::getExtensionType(), llvm::SelectionDAG::getExtLoad(), llvm::MachineMemOperand::getFlags(), llvm::SDNode::getFlags(), llvm::GCNSubtarget::getGeneration(), llvm::MachineFunction::getInfo(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::GCNSubtarget::getMaxPrivateElementSize(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::LoadSDNode::getOffset(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::GCNSubtarget::getScalarizeGlobalBehavior(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSize(), llvm::SelectionDAG::getTarget(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZeroExtendInReg(), AMDGPUAS::GLOBAL_ADDRESS, llvm::SDNodeFlags::hasAllowReciprocal(), llvm::SIMachineFunctionInfo::hasFlatScratchInit(), llvm::AMDGPUSubtarget::hasFP32Denormals(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::SDNode::isDivergent(), llvm::EVT::isFloatingPoint(), llvm::MemSDNode::isInvariant(), llvm::SITargetLowering::isMemOpHasNoClobberedMemOperand(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm_unreachable, llvm::MipsISD::Lo, llvm::SPII::Load, AMDGPUAS::LOCAL_ADDRESS, llvm::ISD::NON_EXTLOAD, llvm::TargetMachine::Options, AMDGPUAS::PRIVATE_ADDRESS, llvm::AMDGPUISD::RCP, llvm::AMDGPUISD::RSQ, llvm::TargetLowering::scalarizeVectorLoad(), llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::AMDGPUSubtarget::SOUTHERN_ISLANDS, llvm::AMDGPUTargetLowering::SplitVectorLoad(), llvm::ISD::TRUNCATE, llvm::ISD::UNINDEXED, llvm::TargetOptions::UnsafeFPMath, llvm::GCNSubtarget::useDS128(), llvm::MVT::v2i32, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

getMad64_32()

static SDValue getMad64_32 ( SelectionDAG &  DAG, const SDLoc &  SL, EVT  VT, SDValue  N0, SDValue  N1, SDValue  N2, bool  Signed  )

static

Definition at line 8421 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::ADDCARRY, llvm::AfterLegalizeDAG, llvm::TargetOptions::AllowFPOpFusion, llvm::ISD::ANY_EXTEND, llvm::AMDGPU::HSAMD::Kernel::Key::Args, assert(), llvm::APFloatBase::cmpGreaterThan, llvm::APFloatBase::cmpLessThan, llvm::APFloatBase::cmpUnordered, llvm::APFloat::compare(), llvm::AMDGPUISD::CVT_F32_UBYTE0, llvm::dyn_cast(), llvm::GCNSubtarget::enableDX10Clamp(), llvm::ISD::EXTRACT_VECTOR_ELT, F(), llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::ISD::FABS, llvm::ISD::FADD, llvm::FPOpFusion::Fast, llvm::AMDGPUISD::FDOT2, llvm::ISD::FMA, llvm::ISD::FNEG, llvm::AMDGPUISD::FP_CLASS, llvm::ISD::FP_EXTEND, llvm::APInt::getBitsSet(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::APFloat::getSemantics(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ConstantFPSDNode::getValueAPF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::APFloat::getZero(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::ConstantSDNode::getZExtValue(), llvm::AMDGPUSubtarget::has16BitInsts(), llvm::SDNodeFlags::hasAllowContract(), llvm::GCNSubtarget::hasDotInsts(), llvm::GCNSubtarget::hasMad64_32(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::ConstantSDNode::isAllOnesValue(), isBoolSGPR(), llvm::APFloat::isInfinity(), llvm::APFloat::isNegative(), llvm::ConstantSDNode::isNullValue(), llvm::EVT::isVector(), llvm::AMDGPUISD::MAD_I64_I32, llvm::AMDGPUISD::MAD_U64_U32, llvm::BitmaskEnumDetail::Mask(), llvm::ISD::MUL, llvm::SIInstrFlags::N_INFINITY, llvm::SIInstrFlags::N_NORMAL, llvm::SIInstrFlags::N_SUBNORMAL, llvm::SIInstrFlags::N_ZERO, llvm::AMDGPUTargetLowering::numBitsSigned(), llvm::AMDGPUTargetLowering::numBitsUnsigned(), llvm::TargetMachine::Options, llvm::SIInstrFlags::P_INFINITY, llvm::SIInstrFlags::P_NORMAL, llvm::SIInstrFlags::P_SUBNORMAL, llvm::SIInstrFlags::P_ZERO, llvm::ISD::SELECT, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETONE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::SUBCARRY, std::swap(), llvm::ISD::TRUNCATE, llvm::TargetOptions::UnsafeFPMath, llvm::MVT::v2f16, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

getMOVRELDPseudo()

static unsigned getMOVRELDPseudo ( const SIRegisterInfo &  TRI, const TargetRegisterClass *  VecRC  )

static

Definition at line 3142 of file SIISelLowering.cpp.

References llvm_unreachable.

Referenced by emitIndirectDst().

getPermuteMask()

static uint32_t getPermuteMask ( SelectionDAG &  DAG, SDValue  V  )

static

Definition at line 7221 of file SIISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::AssertZext, llvm::AMDGPUISD::BFE_U32, llvm::ISD::BITCAST, llvm::tgtok::Bits, llvm::ISD::BUILD_VECTOR, C, llvm::countPopulation(), llvm::APInt::countTrailingZeros(), llvm::dyn_cast(), llvm::MipsISD::Ext, llvm::ISD::FABS, llvm::AMDGPUISD::FP_CLASS, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), getConstantPermuteMask(), llvm::SelectionDAG::getContext(), llvm::GCNSubtarget::getInstrInfo(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SITargetLowering::getSubtarget(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::SDValue::hasOneUse(), I, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, isBoolSGPR(), llvm::SDNode::isDivergent(), llvm::ConstantFPSDNode::isInfinity(), llvm::ConstantFPSDNode::isNegative(), llvm::isShiftedMask_64(), llvm::BitmaskEnumDetail::Mask(), llvm::SIInstrFlags::N_INFINITY, llvm::SIInstrFlags::N_NORMAL, llvm::SIInstrFlags::N_SUBNORMAL, llvm::SIInstrFlags::N_ZERO, llvm::ISD::OR, llvm::SIInstrFlags::P_INFINITY, llvm::SIInstrFlags::P_NORMAL, llvm::SIInstrFlags::P_SUBNORMAL, llvm::SIInstrFlags::P_ZERO, llvm::AMDGPUISD::PERM, llvm::SIInstrInfo::pseudoToMCOpcode(), llvm::SIInstrFlags::Q_NAN, llvm::SIInstrFlags::S_NAN, llvm::ISD::SETCC, llvm::ISD::SETO, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, Split(), llvm::AMDGPUTargetLowering::split64BitValue(), llvm::ISD::SRL, std::swap(), TII, llvm::MVT::v2i32, X, llvm::ISD::XOR, Y, and llvm::ISD::ZERO_EXTEND.

getSplatConstantFP()

static ConstantFPSDNode* getSplatConstantFP ( SDValue  Op )

static

Definition at line 8048 of file SIISelLowering.cpp.

References llvm::APFloat::bitcastToAPInt(), C, llvm::AMDGPUISD::CLAMP, llvm::APFloatBase::cmpGreaterThan, llvm::APFloat::compare(), llvm::GCNSubtarget::enableDX10Clamp(), llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::AMDGPUISD::FMAX_LEGACY, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMED3, llvm::AMDGPUISD::FMIN_LEGACY, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::GCNSubtarget::getInstrInfo(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SITargetLowering::getSubtarget(), llvm::ConstantFPSDNode::getValueAPF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::AMDGPUSubtarget::has16BitInsts(), llvm::GCNSubtarget::hasMed3_16(), llvm::GCNSubtarget::hasMin3Max3_16(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::AMDGPUSubtarget::hasVOP3PInsts(), llvm::MVT::i16, llvm::ConstantFPSDNode::isExactlyValue(), llvm::SIInstrInfo::isInlineConstant(), llvm::SelectionDAG::isKnownNeverSNaN(), llvm::EVT::isVector(), minMaxOpcToMin3Max3Opc(), llvm::ISD::SMAX, llvm::ISD::SMIN, TII, llvm::ISD::UMAX, llvm::ISD::UMIN, and llvm::MVT::v2f16.

isBoolSGPR()

static bool isBoolSGPR ( SDValue  V )

static

Definition at line 7184 of file SIISelLowering.cpp.

References llvm::ISD::AND, llvm::AMDGPUISD::FP_CLASS, llvm::SDValue::getOpcode(), llvm::SDValue::getValueType(), llvm::MVT::i1, llvm::ISD::OR, llvm::ISD::SETCC, and llvm::ISD::XOR.

Referenced by getMad64_32(), and getPermuteMask().

isClampZeroToOne()

static bool isClampZeroToOne ( SDValue  A, SDValue  B  )

static

Definition at line 8180 of file SIISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetOptions::AllowFPOpFusion, llvm::AMDGPUTargetLowering::allUsesHaveSourceMods(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::AMDGPUISD::CLAMP, llvm::dyn_cast(), E, llvm::GCNSubtarget::enableDX10Clamp(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f32, llvm::ISD::FABS, llvm::ISD::FADD, llvm::FPOpFusion::Fast, llvm::ISD::FMA, llvm::ISD::FMAD, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::ISD::FMUL, llvm::ISD::FNEG, llvm::ISD::FSUB, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::AMDGPUTargetLowering::getEquivalentMemType(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelectCC(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTarget(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDNodeFlags::hasAllowContract(), llvm::GCNSubtarget::hasFP16Denormals(), llvm::AMDGPUSubtarget::hasFP32Denormals(), llvm::SDValue::hasOneUse(), I, llvm::MVT::i32, llvm::MipsISD::Ins, llvm::EVT::isByteSized(), llvm::SITargetLowering::isFMAFasterThanFMulAndFAdd(), llvm::SDValue::isUndef(), llvm::TargetMachine::Options, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SETEQ, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SRL, std::swap(), llvm::ISD::TRUNCATE, llvm::ISD::UMAX, llvm::ISD::UMIN, and llvm::TargetOptions::UnsafeFPMath.

isCopyFromRegOfInlineAsm()

static LLVM_ATTRIBUTE_UNUSED bool isCopyFromRegOfInlineAsm ( const SDNode *  N )

static

Definition at line 9590 of file SIISelLowering.cpp.

References assert(), llvm::ISD::CopyFromReg, llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), and llvm::ISD::INLINEASM.

Referenced by llvm::SITargetLowering::isSDNodeSourceOfDivergence().

isFlatGlobalAddrSpace()

static bool isFlatGlobalAddrSpace ( unsigned  AS )

static

Definition at line 1205 of file SIISelLowering.cpp.

References AMDGPUAS::CONSTANT_ADDRESS, AMDGPUAS::FLAT_ADDRESS, and AMDGPUAS::GLOBAL_ADDRESS.

Referenced by getFPTernOp(), and llvm::SITargetLowering::isNoopAddrSpaceCast().

isFrameIndexOp()

static bool isFrameIndexOp ( SDValue  Op )

static

Definition at line 9181 of file SIISelLowering.cpp.

References llvm::ISD::AssertZext, llvm::SDValue::getOpcode(), and llvm::SDValue::getOperand().

Referenced by llvm::SITargetLowering::legalizeTargetIndependentNode().

loadM0FromVGPR()

static MachineBasicBlock::iterator loadM0FromVGPR ( const SIInstrInfo *  TII, MachineBasicBlock &  MBB, MachineInstr &  MI, unsigned  InitResultReg, unsigned  PhiReg, int  Offset, bool  UseGPRIdxMode, bool  IsIndirectSrc  )

static

Definition at line 2940 of file SIISelLowering.cpp.

References llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), emitLoadM0FromVGPRLoop(), llvm::MachineBasicBlock::end(), llvm::MachineInstr::getDebugLoc(), llvm::SIInstrInfo::getNamedOperand(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), I, llvm::MachineFunction::insert(), MRI, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by emitIndirectDst(), and emitIndirectSrc().

lowerFCMPIntrinsic()

static SDValue lowerFCMPIntrinsic ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 3780 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::MVT::f16, llvm::MVT::f32, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getCondCode(), llvm::getFCmpCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::AMDGPUISD::SETCC.

Referenced by parseTexFail().

lowerICMPIntrinsic()

static SDValue lowerICMPIntrinsic ( const SITargetLowering &  TLI, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 3746 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::SelectionDAG::getCondCode(), llvm::getICmpCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i16, llvm::MVT::i32, llvm::CmpInst::isSigned(), llvm::TargetLoweringBase::isTypeLegal(), llvm::AMDGPUISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by parseTexFail().

mayTailCallThisCC()

static bool mayTailCallThisCC ( CallingConv::ID  CC )

static

Return true if we might ever do TCO for calls with this calling convention.

Definition at line 2351 of file SIISelLowering.cpp.

References llvm::CallingConv::C, and canGuaranteeTCO().

Referenced by llvm::SITargetLowering::isEligibleForTailCallOptimization().

memVTFromAggregate()

static MVT memVTFromAggregate ( Type *  Ty )

static

Definition at line 817 of file SIISelLowering.cpp.

References assert(), llvm::Type::FloatTyID, getBitWidth(), llvm::Type::getContainedType(), llvm::SequentialType::getElementType(), llvm::SequentialType::getNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::getVT(), llvm::Type::HalfTyID, llvm::Type::IntegerTyID, llvm::Type::isIntegerTy(), llvm::Type::isStructTy(), llvm::Type::isVectorTy(), llvm_unreachable, and llvm::Log2_32_Ceil().

Referenced by llvm::SITargetLowering::getTgtMemIntrinsic().

minMaxOpcToMin3Max3Opc()

static unsigned minMaxOpcToMin3Max3Opc ( unsigned  Opc )

static

Definition at line 7989 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::AMDGPUISD::FMAX3, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::AMDGPUISD::FMIN3, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::GCNSubtarget::hasMed3_16(), llvm::MVT::i16, llvm::MVT::i32, llvm_unreachable, llvm::APInt::sge(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SMAX, llvm::AMDGPUISD::SMAX3, llvm::AMDGPUISD::SMED3, llvm::ISD::SMIN, llvm::AMDGPUISD::SMIN3, llvm::ISD::TRUNCATE, llvm::APInt::uge(), llvm::ISD::UMAX, llvm::AMDGPUISD::UMAX3, llvm::AMDGPUISD::UMED3, llvm::ISD::UMIN, llvm::AMDGPUISD::UMIN3, and llvm::ISD::ZERO_EXTEND.

Referenced by getSplatConstantFP().

parseCachePolicy()

static bool parseCachePolicy ( SDValue  CachePolicy, SelectionDAG &  DAG, SDValue *  GLC, SDValue *  SLC  )

static

Definition at line 4647 of file SIISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::MVT::i32.

Referenced by parseTexFail().

parseTexFail()

static bool parseTexFail ( SDValue  TexFailCtrl, SelectionDAG &  DAG, SDValue *  TFE, SDValue *  LWE, bool &  IsTexFail  )

static

Definition at line 4750 of file SIISelLowering.cpp.

References llvm::Intrinsic::amdgcn_class, llvm::Intrinsic::amdgcn_cos, llvm::Intrinsic::amdgcn_cvt_pk_i16, llvm::Intrinsic::amdgcn_cvt_pk_u16, llvm::Intrinsic::amdgcn_cvt_pknorm_i16, llvm::Intrinsic::amdgcn_cvt_pknorm_u16, llvm::Intrinsic::amdgcn_cvt_pkrtz, llvm::Intrinsic::amdgcn_dispatch_id, llvm::Intrinsic::amdgcn_dispatch_ptr, llvm::Intrinsic::amdgcn_div_fixup, llvm::Intrinsic::amdgcn_div_fmas, llvm::Intrinsic::amdgcn_div_scale, llvm::Intrinsic::amdgcn_fcmp, llvm::Intrinsic::amdgcn_fdiv_fast, llvm::Intrinsic::amdgcn_fdot2, llvm::Intrinsic::amdgcn_fmed3, llvm::Intrinsic::amdgcn_fmul_legacy, llvm::Intrinsic::amdgcn_fract, llvm::Intrinsic::amdgcn_icmp, llvm::Intrinsic::amdgcn_implicit_buffer_ptr, llvm::Intrinsic::amdgcn_implicitarg_ptr, llvm::Intrinsic::amdgcn_interp_mov, llvm::Intrinsic::amdgcn_interp_p1, llvm::Intrinsic::amdgcn_interp_p2, llvm::Intrinsic::amdgcn_kernarg_segment_ptr, llvm::Intrinsic::amdgcn_ldexp, llvm::Intrinsic::amdgcn_log_clamp, llvm::Intrinsic::amdgcn_queue_ptr, llvm::Intrinsic::amdgcn_rcp, llvm::Intrinsic::amdgcn_rcp_legacy, llvm::Intrinsic::amdgcn_rsq, llvm::Intrinsic::amdgcn_rsq_clamp, llvm::Intrinsic::amdgcn_rsq_legacy, llvm::Intrinsic::amdgcn_s_buffer_load, llvm::Intrinsic::amdgcn_sbfe, llvm::Intrinsic::amdgcn_sffbh, llvm::Intrinsic::amdgcn_sin, llvm::Intrinsic::amdgcn_trig_preop, llvm::Intrinsic::amdgcn_ubfe, llvm::Intrinsic::amdgcn_workgroup_id_x, llvm::Intrinsic::amdgcn_workgroup_id_y, llvm::Intrinsic::amdgcn_workgroup_id_z, llvm::Intrinsic::amdgcn_workitem_id_x, llvm::Intrinsic::amdgcn_workitem_id_y, llvm::Intrinsic::amdgcn_workitem_id_z, assert(), llvm::AMDGPUISD::BFE_I32, llvm::AMDGPUISD::BFE_U32, llvm::ISD::BITCAST, llvm::AMDGPUISD::BUFFER_LOAD, llvm::ISD::CONCAT_VECTORS, constructRetValue(), llvm::SITargetLowering::copyToM0(), llvm::AMDGPUISD::COS_HW, llvm::countPopulation(), llvm::AMDGPUISD::CVT_PK_I16_I32, llvm::AMDGPUISD::CVT_PK_U16_U32, llvm::AMDGPUISD::CVT_PKNORM_I16_F32, llvm::AMDGPUISD::CVT_PKNORM_U16_F32, llvm::AMDGPUISD::CVT_PKRTZ_F16_F32, llvm::LLVMContext::diagnose(), llvm::AMDGPUFunctionArgInfo::DISPATCH_ID, llvm::AMDGPUFunctionArgInfo::DISPATCH_PTR, llvm::AMDGPUISD::DIV_FIXUP, llvm::AMDGPUISD::DIV_FMAS, llvm::AMDGPUISD::DIV_SCALE, llvm::dyn_cast(), emitNonHSAIntrinsicError(), emitRemovedIntrinsicError(), llvm::SelectionDAG::ExtractVectorElements(), llvm::MVT::f16, llvm::MVT::f32, llvm::AMDGPUISD::FDOT2, llvm::AMDGPUISD::FFBH_I32, llvm::ISD::FMAXNUM, llvm::AMDGPUISD::FMED3, llvm::ISD::FMINNUM, llvm::AMDGPUISD::FMUL_LEGACY, llvm::AMDGPUISD::FP_CLASS, llvm::AMDGPUISD::FRACT, llvm::SelectionDAG::getBitcast(), getBuildDwordsVector(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SDLoc::getDebugLoc(), llvm::SelectionDAG::getEntryNode(), llvm::Type::getFltSemantics(), llvm::MachineFunction::getFunction(), llvm::GCNSubtarget::getGeneration(), llvm::MachineFunction::getInfo(), llvm::APFloat::getLargest(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getMergeValues(), llvm::AMDGPU::getMIMGBaseOpcodeInfo(), llvm::AMDGPU::getMIMGDimInfo(), llvm::AMDGPU::getMIMGLZMappingInfo(), llvm::AMDGPU::getMIMGOpcode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSize(), llvm::SITargetLowering::getSubtarget(), llvm::MachineFunction::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::SI::KernelInputOffsets::GLOBAL_SIZE_X, llvm::SI::KernelInputOffsets::GLOBAL_SIZE_Y, llvm::SI::KernelInputOffsets::GLOBAL_SIZE_Z, llvm::MVT::Glue, llvm::GCNSubtarget::hasUnpackedD16VMem(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR, llvm::AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR, llvm::AMDGPUISD::INTERP_MOV, llvm::AMDGPUISD::INTERP_P1, llvm::AMDGPUISD::INTERP_P2, llvm::ConstantSDNode::isAllOnesValue(), llvm::AMDGPUSubtarget::isAmdHsaOrMesa(), llvm::AMDGPUSubtarget::isAmdHsaOS(), llvm::SDNode::isDivergent(), llvm::AMDGPUMachineFunction::isEntryFunction(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR, llvm::AMDGPUISD::LDEXP, llvm::SPII::Load, llvm::AMDGPUTargetLowering::loadInputValue(), llvm::SI::KernelInputOffsets::LOCAL_SIZE_X, llvm::SI::KernelInputOffsets::LOCAL_SIZE_Y, llvm::SI::KernelInputOffsets::LOCAL_SIZE_Z, llvm::Log2_32_Ceil(), lowerFCMPIntrinsic(), lowerICMPIntrinsic(), llvm::MachineMemOperand::MODereferenceable, llvm::MachineMemOperand::MOInvariant, llvm::MachineMemOperand::MOLoad, llvm::SI::KernelInputOffsets::NGROUPS_X, llvm::SI::KernelInputOffsets::NGROUPS_Y, llvm::SI::KernelInputOffsets::NGROUPS_Z, parseCachePolicy(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::AMDGPUFunctionArgInfo::QUEUE_PTR, llvm::Intrinsic::r600_read_global_size_x, llvm::Intrinsic::r600_read_global_size_y, llvm::Intrinsic::r600_read_global_size_z, llvm::Intrinsic::r600_read_local_size_x, llvm::Intrinsic::r600_read_local_size_y, llvm::Intrinsic::r600_read_local_size_z, llvm::Intrinsic::r600_read_ngroups_x, llvm::Intrinsic::r600_read_ngroups_y, llvm::Intrinsic::r600_read_ngroups_z, llvm::Intrinsic::r600_read_tgid_x, llvm::Intrinsic::r600_read_tgid_y, llvm::Intrinsic::r600_read_tgid_z, llvm::Intrinsic::r600_read_tidig_x, llvm::Intrinsic::r600_read_tidig_y, llvm::Intrinsic::r600_read_tidig_z, llvm::AMDGPUISD::RCP, llvm::AMDGPUISD::RCP_LEGACY, llvm::AMDGPUISD::RSQ, llvm::AMDGPUISD::RSQ_CLAMP, llvm::AMDGPUISD::RSQ_LEGACY, llvm::AMDGPUISD::SBUFFER_LOAD, llvm::ISD::SCALAR_TO_VECTOR, llvm::SelectionDAG::setNodeMemRefs(), llvm::AMDGPUISD::SIN_HW, llvm::ARM_MB::ST, llvm::AMDGPUISD::TRIG_PREOP, llvm::RegState::Undef, llvm::MVT::v16i32, llvm::MVT::v2f16, llvm::MVT::v2i16, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i32, llvm::SDNode::value_begin(), llvm::SDNode::value_end(), llvm::AMDGPUSubtarget::VOLCANIC_ISLANDS, llvm::AMDGPUFunctionArgInfo::WORKGROUP_ID_X, llvm::AMDGPUFunctionArgInfo::WORKGROUP_ID_Y, and llvm::AMDGPUFunctionArgInfo::WORKGROUP_ID_Z.

processShaderInputArgs()

static void processShaderInputArgs ( SmallVectorImpl< ISD::InputArg > &  Splits, CallingConv::ID  CallConv, ArrayRef< ISD::InputArg >  Ins, BitVector &  Skipped, FunctionType *  FType, SIMachineFunctionInfo *  Info  )

static

Definition at line 1434 of file SIISelLowering.cpp.

References llvm::CallingConv::AMDGPU_PS, assert(), E, llvm::ISD::InputArg::Flags, llvm::ISD::InputArg::getOrigArgIndex(), llvm::MVT::getScalarSizeInBits(), I, llvm::ISD::ArgFlagsTy::isByVal(), llvm::ISD::ArgFlagsTy::isInReg(), llvm::SIMachineFunctionInfo::isPSInputAllocated(), llvm::ISD::ArgFlagsTy::isSplit(), llvm::ISD::ArgFlagsTy::isSplitEnd(), llvm::MVT::isVector(), llvm::SIMachineFunctionInfo::markPSInputAllocated(), llvm::SIMachineFunctionInfo::markPSInputEnabled(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::BitVector::set(), llvm::ArrayRef< T >::size(), llvm::ISD::InputArg::Used, and llvm::ISD::InputArg::VT.

Referenced by llvm::SITargetLowering::LowerFormalArguments().

reservePrivateMemoryRegs()

static void reservePrivateMemoryRegs ( const TargetMachine &  TM, MachineFunction &  MF, const SIRegisterInfo &  TRI, SIMachineFunctionInfo &  Info  )

static

Definition at line 1713 of file SIISelLowering.cpp.

References llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::TargetMachine::getOptLevel(), llvm::SIMachineFunctionInfo::getPreloadedReg(), llvm::MachineFunction::getSubtarget(), llvm::MachineFrameInfo::hasCalls(), llvm::MachineFrameInfo::hasStackObjects(), llvm::AMDGPUSubtarget::isAmdHsaOrMesa(), llvm::CodeGenOpt::None, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER, llvm::AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET, llvm::SIRegisterInfo::reservedPrivateSegmentBufferReg(), llvm::SIRegisterInfo::reservedPrivateSegmentWaveByteOffsetReg(), llvm::SIMachineFunctionInfo::setHasNonSpillStackObjects(), llvm::SIMachineFunctionInfo::setScratchRSrcReg(), llvm::SIMachineFunctionInfo::setScratchWaveOffsetReg(), and llvm::ARM_MB::ST.

Referenced by llvm::SITargetLowering::finalizeLowering().

setM0ToIndexFromSGPR()

static bool setM0ToIndexFromSGPR ( const SIInstrInfo *  TII, MachineRegisterInfo &  MRI, MachineInstr &  MI, int  Offset, bool  UseGPRIdxMode, bool  IsIndirectSrc  )

static

Definition at line 3012 of file SIISelLowering.cpp.

References llvm::ARM_AM::add, llvm::MachineInstrBuilder::addImm(), assert(), llvm::BuildMI(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::VGPRIndexMode::DST_ENABLE, llvm::MachineInstr::getDebugLoc(), llvm::SIInstrInfo::getNamedOperand(), llvm::MachineInstr::getOperand(), llvm::MachineInstr::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::SIInstrInfo::getRegisterInfo(), I, llvm::SIRegisterInfo::isSGPRClass(), llvm::RegState::Kill, llvm::MachineOperand::setIsUndef(), and llvm::VGPRIndexMode::SRC0_ENABLE.

Referenced by emitIndirectDst(), and emitIndirectSrc().

STATISTIC()

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"
	)

SubIdx2Lane()

static unsigned SubIdx2Lane ( unsigned  Idx )

static

Helper function for adjustWritemask.

Definition at line 9012 of file SIISelLowering.cpp.

vectorEltWillFoldAway()

static bool vectorEltWillFoldAway ( SDValue  Op )

static

Definition at line 7897 of file SIISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::ISD::FCANONICALIZE, llvm::ISD::FMAXNUM, llvm::ISD::FMINNUM, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstantFP(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::APFloat::getQNaN(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MipsISD::Hi, I, llvm::SITargetLowering::isCanonicalized(), llvm::isConstOrConstSplatFP(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), isUndef(), llvm::MipsISD::Lo, and llvm::MVT::v2f16.

Variable Documentation

AssumeFrameIndexHighZeroBits

cl::opt<unsigned> AssumeFrameIndexHighZeroBits("amdgpu-frame-index-zero-bits", cl::desc("High bits of frame index assumed to be zero"), cl::init(5), cl::ReallyHidden)

static

Referenced by llvm::SITargetLowering::computeKnownBitsForFrameIndex(), and llvm::SITargetLowering::LowerFormalArguments().

EnableVGPRIndexMode

cl::opt<bool> EnableVGPRIndexMode("amdgpu-vgpr-index-mode", cl::desc("Use GPR indexing mode instead of movrel for vector indexing"), cl::init(false))

static

Referenced by emitIndirectDst(), and emitIndirectSrc().