AArch64ISelLowering.cpp File Reference

#include "AArch64ISelLowering.h"
#include "AArch64CallingConvention.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/OperandTraits.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MachineValueType.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <tuple>
#include <utility>
#include <vector>
#include "AArch64GenCallingConv.inc"

Go to the source code of this file.

Classes
struct	GenericSetCCInfo
	Helper structure to keep track of ISD::SET_CC operands. More...
struct	AArch64SetCCInfo
	Helper structure to keep track of a SET_CC lowered into AArch64 code. More...
union	SetCCInfo
	Helper structure to keep track of SetCC information. More...
struct	SetCCInfoAndKind
	Helper structure to be able to read SetCC information. More...

Macros
#define	DEBUG_TYPE   "aarch64-lower"

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")
	STATISTIC (NumShiftInserts, "Number of vector shift inserts")
	STATISTIC (NumOptimizedImms, "Number of times immediates were optimized")
static bool	optimizeLogicalImm (SDValue Op, unsigned Size, uint64_t Imm, const APInt &Demanded, TargetLowering::TargetLoweringOpt &TLO, unsigned NewOpc)
static AArch64CC::CondCode	changeIntCCToAArch64CC (ISD::CondCode CC)
	changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC More...
static void	changeFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
	changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC. More...
static void	changeFPCCToANDAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2)
	Convert a DAG fp condition code to an AArch64 CC. More...
static void	changeVectorFPCCToAArch64CC (ISD::CondCode CC, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
	changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions. More...
static bool	isLegalArithImmed (uint64_t C)
static bool	isCMN (SDValue Op, ISD::CondCode CC)
static SDValue	emitComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG)
static unsigned	getCmpOperandFoldingProfit (SDValue Op)
	Returns how profitable it is to fold a comparison's operand's shift and/or extension operations. More...
static SDValue	getAArch64Cmp (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AArch64cc, SelectionDAG &DAG, const SDLoc &dl)
static std::pair< SDValue, SDValue >	getAArch64XALUOOp (AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG)
static bool	isOverflowIntrOpRes (SDValue Op)
static SDValue	LowerXOR (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerADDC_ADDE_SUBC_SUBE (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerXALUO (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerPREFETCH (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerVectorFP_TO_INT (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerVectorINT_TO_FP (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerBITCAST (SDValue Op, SelectionDAG &DAG)
static EVT	getExtensionTo64Bits (const EVT &OrigVT)
static SDValue	addRequiredExtensionForVectorMULL (SDValue N, SelectionDAG &DAG, const EVT &OrigTy, const EVT &ExtTy, unsigned ExtOpcode)
static bool	isExtendedBUILD_VECTOR (SDNode *N, SelectionDAG &DAG, bool isSigned)
static SDValue	skipExtensionForVectorMULL (SDNode *N, SelectionDAG &DAG)
static bool	isSignExtended (SDNode *N, SelectionDAG &DAG)
static bool	isZeroExtended (SDNode *N, SelectionDAG &DAG)
static bool	isAddSubSExt (SDNode *N, SelectionDAG &DAG)
static bool	isAddSubZExt (SDNode *N, SelectionDAG &DAG)
static SDValue	LowerMUL (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerTruncateVectorStore (SDLoc DL, StoreSDNode *ST, EVT VT, EVT MemVT, SelectionDAG &DAG)
static bool	canGuaranteeTCO (CallingConv::ID CC)
	Return true if the calling convention is one that we can guarantee TCO for. More...
static bool	mayTailCallThisCC (CallingConv::ID CC)
	Return true if we might ever do TCO for calls with this calling convention. More...
static SDValue	getEstimate (const AArch64Subtarget *ST, unsigned Opcode, SDValue Operand, SelectionDAG &DAG, int &ExtraSteps)
static SDValue	WidenVector (SDValue V64Reg, SelectionDAG &DAG)
	WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class. More...
static unsigned	getExtFactor (SDValue &V)
	getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction. More...
static SDValue	NarrowVector (SDValue V128Reg, SelectionDAG &DAG)
	NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 register class. More...
static bool	isSingletonEXTMask (ArrayRef< int > M, EVT VT, unsigned &Imm)
static bool	isEXTMask (ArrayRef< int > M, EVT VT, bool &ReverseEXT, unsigned &Imm)
static bool	isREVMask (ArrayRef< int > M, EVT VT, unsigned BlockSize)
	isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize. More...
static bool	isZIPMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool	isUZPMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool	isTRNMask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool	isZIP_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
	isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More...
static bool	isUZP_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
	isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More...
static bool	isTRN_v_undef_Mask (ArrayRef< int > M, EVT VT, unsigned &WhichResult)
	isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef". More...
static bool	isINSMask (ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
static bool	isConcatMask (ArrayRef< int > Mask, EVT VT, bool SplitLHS)
static SDValue	tryFormConcatFromShuffle (SDValue Op, SelectionDAG &DAG)
static SDValue	GeneratePerfectShuffle (unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl)
	GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle. More...
static SDValue	GenerateTBL (SDValue Op, ArrayRef< int > ShuffleMask, SelectionDAG &DAG)
static unsigned	getDUPLANEOp (EVT EltType)
static bool	resolveBuildVector (BuildVectorSDNode *BVN, APInt &CnstBits, APInt &UndefBits)
static SDValue	tryAdvSIMDModImm64 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImm32 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue	tryAdvSIMDModImm16 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits, const SDValue *LHS=nullptr)
static SDValue	tryAdvSIMDModImm321s (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImm8 (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static SDValue	tryAdvSIMDModImmFP (unsigned NewOp, SDValue Op, SelectionDAG &DAG, const APInt &Bits)
static bool	isAllConstantBuildVector (const SDValue &PotentialBVec, uint64_t &ConstVal)
static unsigned	getIntrinsicID (const SDNode *N)
static SDValue	tryLowerToSLI (SDNode *N, SelectionDAG &DAG)
static SDValue	NormalizeBuildVector (SDValue Op, SelectionDAG &DAG)
static SDValue	ConstantBuildVector (SDValue Op, SelectionDAG &DAG)
static bool	getVShiftImm (SDValue Op, unsigned ElementBits, int64_t &Cnt)
	getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value. More...
static bool	isVShiftLImm (SDValue Op, EVT VT, bool isLong, int64_t &Cnt)
	isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation. More...
static bool	isVShiftRImm (SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
	isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation. More...
static SDValue	EmitVectorComparison (SDValue LHS, SDValue RHS, AArch64CC::CondCode CC, bool NoNans, EVT VT, const SDLoc &dl, SelectionDAG &DAG)
static SDValue	getReductionSDNode (unsigned Op, SDLoc DL, SDValue ScalarOp, SelectionDAG &DAG)
static bool	memOpAlign (unsigned DstAlign, unsigned SrcAlign, unsigned AlignCheck)
static SDValue	foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
	Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0. More...
static SDValue	performIntegerAbsCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performXorCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performMulCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performVectorCompareAndMaskUnaryOpCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performIntToFpCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performFpToIntCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
	Fold a floating-point multiply by power of two into floating-point to fixed-point conversion. More...
static SDValue	performFDivCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
	Fold a floating-point divide by power of two into fixed-point to floating-point conversion. More...
static bool	findEXTRHalf (SDValue N, SDValue &Src, uint32_t &ShiftAmount, bool &FromHi)
	An EXTR instruction is made up of two shifts, ORed together. More...
static SDValue	tryCombineToEXTR (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
	EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low pair. More...
static SDValue	tryCombineToBSL (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performORCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performSRLCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performBitcastCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performConcatVectorsCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryCombineFixedPointConvert (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryExtendDUPToExtractHigh (SDValue N, SelectionDAG &DAG)
static bool	isEssentiallyExtractSubvector (SDValue N)
static bool	isSetCC (SDValue Op, SetCCInfoAndKind &SetCCInfo)
	Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one. More...
static bool	isSetCCOrZExtSetCC (const SDValue &Op, SetCCInfoAndKind &Info)
static SDValue	performSetccAddFolding (SDNode *Op, SelectionDAG &DAG)
static SDValue	performAddSubLongCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryCombineLongOpWithDup (unsigned IID, SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	tryCombineShiftImm (unsigned IID, SDNode *N, SelectionDAG &DAG)
static SDValue	tryCombineCRC32 (unsigned Mask, SDNode *N, SelectionDAG &DAG)
static SDValue	combineAcrossLanesIntrinsic (unsigned Opc, SDNode *N, SelectionDAG &DAG)
static SDValue	performIntrinsicCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue	performExtendCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	splitStoreSplat (SelectionDAG &DAG, StoreSDNode &St, SDValue SplatVal, unsigned NumVecElts)
static SDValue	replaceZeroVectorStore (SelectionDAG &DAG, StoreSDNode &St)
	Replace a splat of zeros to a vector store by scalar stores of WZR/XZR. More...
static SDValue	replaceSplatVectorStore (SelectionDAG &DAG, StoreSDNode &St)
	Replace a splat of a scalar to a vector store by scalar stores of the scalar value. More...
static SDValue	splitStores (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performPostLD1Combine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, bool IsLaneOp)
	Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R. More...
static bool	performTBISimplification (SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
	Simplify Addr given that the top byte of it is ignored by HW during address translation. More...
static SDValue	performSTORECombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static SDValue	performNEONPostLDSTCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
	Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates. More...
static bool	checkValueWidth (SDValue V, unsigned width, ISD::LoadExtType &ExtType)
static bool	isEquivalentMaskless (unsigned CC, unsigned width, ISD::LoadExtType ExtType, int AddConstant, int CompConstant)
static SDValue	performCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, unsigned CCIndex, unsigned CmpIndex)
static SDValue	performBRCONDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	getTestBitOperand (SDValue Op, unsigned &Bit, bool &Invert, SelectionDAG &DAG)
static SDValue	performTBZCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue	performVSelectCombine (SDNode *N, SelectionDAG &DAG)
static SDValue	performSelectCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
	A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar. More...
static SDValue	performNVCASTCombine (SDNode *N)
	Get rid of unnecessary NVCASTs (that don't change the type). More...
static SDValue	performGlobalAddressCombine (SDNode *N, SelectionDAG &DAG, const AArch64Subtarget *Subtarget, const TargetMachine &TM)
static void	ReplaceBITCASTResults (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static void	ReplaceReductionResults (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, unsigned InterOp, unsigned AcrossOp)
static std::pair< SDValue, SDValue >	splitInt128 (SDValue N, SelectionDAG &DAG)
static SDValue	createGPRPairNode (SelectionDAG &DAG, SDValue V)
static void	ReplaceCMP_SWAP_128Results (SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const AArch64Subtarget *Subtarget)
static Value *	UseTlsOffset (IRBuilder<> &IRB, unsigned Offset)
static SDValue	emitConditionalComparison (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue CCOp, AArch64CC::CondCode Predicate, AArch64CC::CondCode OutCC, const SDLoc &DL, SelectionDAG &DAG)
	can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags More...
static bool	canEmitConjunction (const SDValue Val, bool &CanNegate, bool &MustBeFirst, bool WillNegate, unsigned Depth=0)
	Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction. More...
static SDValue	emitConjunctionRec (SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, AArch64CC::CondCode Predicate)
	Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops. More...
static SDValue	emitConjunction (SelectionDAG &DAG, SDValue Val, AArch64CC::CondCode &OutCC)
	Emit expression as a conjunction (a series of CCMP/CFCMP ops). More...

Variables
static cl::opt< bool >	EnableAArch64SlrGeneration ("aarch64-shift-insert-generation", cl::Hidden, cl::desc("Allow AArch64 SLI/SRI formation"), cl::init(false))
cl::opt< bool >	EnableAArch64ELFLocalDynamicTLSGeneration ("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))
static cl::opt< bool >	EnableOptimizeLogicalImm ("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true))
static const MVT	MVT_CC = MVT::i32
	Value type used for condition codes. More...

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "aarch64-lower"

Definition at line 91 of file AArch64ISelLowering.cpp.

Function Documentation

addRequiredExtensionForVectorMULL()

static SDValue addRequiredExtensionForVectorMULL ( SDValue  N, SelectionDAG &  DAG, const EVT &  OrigTy, const EVT &  ExtTy, unsigned  ExtOpcode  )

static

Definition at line 2523 of file AArch64ISelLowering.cpp.

References assert(), getExtensionTo64Bits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), and llvm::EVT::is128BitVector().

Referenced by skipExtensionForVectorMULL().

canEmitConjunction()

static bool canEmitConjunction ( const SDValue  Val, bool &  CanNegate, bool &  MustBeFirst, bool  WillNegate, unsigned  Depth = 0  )

static

Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.

See CMP;CCMP matching.

Parameters

CanNegate	Set to true if we can negate the whole sub-tree just by changing the conditions on the SETCC tests. (this means we can call emitConjunctionRec() with Negate==true on this sub-tree)
MustBeFirst	Set to true if this subtree needs to be negated and we cannot do the negation naturally. We are required to emit the subtree first in this case.
WillNegate	Is true if are called when the result of this subexpression must be negated. This happens when the outer expression is an OR. We can use this fact to know that we have a double negation (or (or ...) ...) that can be implemented for free.

Definition at line 1645 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::Depth, llvm::MVT::f128, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::ISD::SETCC.

Referenced by emitConjunction(), and emitConjunctionRec().

canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC )

static

Return true if the calling convention is one that we can guarantee TCO for.

Definition at line 3362 of file AArch64ISelLowering.cpp.

References llvm::CallingConv::Fast.

Referenced by mayTailCallThisCC().

changeFPCCToAArch64CC()

static void changeFPCCToAArch64CC ( ISD::CondCode  CC, AArch64CC::CondCode &  CondCode, AArch64CC::CondCode &  CondCode2  )

static

changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.

Definition at line 1353 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.

Referenced by changeFPCCToANDAArch64CC(), changeVectorFPCCToAArch64CC(), and mayTailCallThisCC().

changeFPCCToANDAArch64CC()

static void changeFPCCToANDAArch64CC ( ISD::CondCode  CC, AArch64CC::CondCode &  CondCode, AArch64CC::CondCode &  CondCode2  )

static

Convert a DAG fp condition code to an AArch64 CC.

This differs from changeFPCCToAArch64CC in that it returns cond codes that should be AND'ed instead of OR'ed.

Definition at line 1416 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, assert(), changeFPCCToAArch64CC(), llvm::AArch64CC::LE, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETONE, llvm::ISD::SETUEQ, and llvm::AArch64CC::VC.

Referenced by emitConjunctionRec().

changeIntCCToAArch64CC()

static AArch64CC::CondCode changeIntCCToAArch64CC ( ISD::CondCode  CC )

static

changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC

Definition at line 1325 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::NE, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, and llvm::ISD::SETULT.

Referenced by emitConjunctionRec(), EmitVectorComparison(), getAArch64Cmp(), and mayTailCallThisCC().

changeVectorFPCCToAArch64CC()

static void changeVectorFPCCToAArch64CC ( ISD::CondCode  CC, AArch64CC::CondCode &  CondCode, AArch64CC::CondCode &  CondCode2, bool &  Invert  )

static

changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.

Fewer operations are available without a real NZCV register, so we have to use less efficient combinations to get the same effect.

Definition at line 1446 of file AArch64ISelLowering.cpp.

References changeFPCCToAArch64CC(), llvm::AArch64CC::GE, llvm::ISD::getSetCCInverse(), LLVM_FALLTHROUGH, llvm::AArch64CC::MI, llvm::ISD::SETO, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::SETUO.

Referenced by EmitVectorComparison().

checkValueWidth()

static bool checkValueWidth ( SDValue  V, unsigned  width, ISD::LoadExtType &  ExtType  )

static

Definition at line 10612 of file AArch64ISelLowering.cpp.

References llvm::abs(), llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::Constant, llvm::LoadSDNode::getExtensionType(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::VTSDNode::getVT(), llvm::MVT::i16, llvm::MVT::i8, llvm::ISD::LOAD, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::TargetConstant, and llvm::ISD::ZEXTLOAD.

Referenced by performCONDCombine().

combineAcrossLanesIntrinsic()

static SDValue combineAcrossLanesIntrinsic ( unsigned  Opc, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 9930 of file AArch64ISelLowering.cpp.

References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getValueType(), and llvm::MVT::i64.

Referenced by performIntrinsicCombine().

ConstantBuildVector()

static SDValue ConstantBuildVector ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 7164 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::dbgs(), llvm::AArch64ISD::DUP, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::AArch64ISD::FMOV, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::BuildVectorSDNode::getConstantSplatNode(), llvm::SelectionDAG::getContext(), getDUPLANEOp(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSplatBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i64, llvm::ISD::INSERT_VECTOR_ELT, isConstant(), llvm::BuildVectorSDNode::isConstant(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::SDValue::isUndef(), LLVM_DEBUG, llvm::AArch64ISD::MOVI, llvm::AArch64ISD::MOVIedit, llvm::AArch64ISD::MOVImsl, llvm::AArch64ISD::MOVIshift, llvm::AArch64ISD::MVNImsl, llvm::AArch64ISD::MVNIshift, NormalizeBuildVector(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::AArch64TargetLowering::ReconstructShuffle(), llvm::SelectionDAG::ReplaceAllUsesWith(), resolveBuildVector(), llvm::ISD::SCALAR_TO_VECTOR, tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), tryAdvSIMDModImmFP(), llvm::AArch64ISD::UZP1, llvm::AArch64ISD::UZP2, llvm::MVT::v16i8, llvm::MVT::v1i64, llvm::MVT::v2f32, llvm::MVT::v2f64, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4f16, llvm::MVT::v4f32, llvm::MVT::v4i16, llvm::MVT::v4i32, llvm::MVT::v8f16, llvm::MVT::v8i16, llvm::MVT::v8i8, and WidenVector().

createGPRPairNode()

static SDValue createGPRPairNode ( SelectionDAG &  DAG, SDValue  V  )

static

Definition at line 11430 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::i128, llvm::MVT::i32, llvm::MVT::i64, llvm::DataLayout::isBigEndian(), llvm::ISD::SRL, std::swap(), and llvm::MVT::Untyped.

Referenced by getAL(), getARClassRegisterMask(), getContiguousRangeOfSetBits(), and ReplaceCMP_SWAP_128Results().

emitComparison()

static SDValue emitComparison ( SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Definition at line 1499 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ADDS, llvm::ISD::AND, llvm::AArch64ISD::ANDS, assert(), llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f32, llvm::AArch64ISD::FCMP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), isCMN(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), llvm::ISD::isUnsignedIntSetCC(), and llvm::AArch64ISD::SUBS.

Referenced by emitConjunctionRec(), getAArch64Cmp(), and mayTailCallThisCC().

emitConditionalComparison()

static SDValue emitConditionalComparison ( SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, SDValue  CCOp, AArch64CC::CondCode  Predicate, AArch64CC::CondCode  OutCC, const SDLoc &  DL, SelectionDAG &  DAG  )

static

can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags

A counterexample is "or (and A B) (and C D)" which translates to not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we can only implement 1 of the inner (not) operations, but not both!Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.

Definition at line 1597 of file AArch64ISelLowering.cpp.

References assert(), llvm::AArch64ISD::CCMN, llvm::AArch64ISD::CCMP, llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f32, llvm::AArch64ISD::FCCMP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValueType(), llvm::MVT::i32, llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::ISD::SUB.

Referenced by emitConjunctionRec().

emitConjunction()

static SDValue emitConjunction ( SelectionDAG &  DAG, SDValue  Val, AArch64CC::CondCode &  OutCC  )

static

Emit expression as a conjunction (a series of CCMP/CFCMP ops).

In some cases this is even possible with OR operations in the expression. See CMP;CCMP matching.

See also

emitConjunctionRec().

Definition at line 1819 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, canEmitConjunction(), and emitConjunctionRec().

Referenced by getAArch64Cmp().

emitConjunctionRec()

static SDValue emitConjunctionRec ( SelectionDAG &  DAG, SDValue  Val, AArch64CC::CondCode &  OutCC, bool  Negate, SDValue  CCOp, AArch64CC::CondCode  Predicate  )

static

Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.

See CMP;CCMP matching. Tries to transform the given i1 producing node Val to a series compare and conditional compare operations.

Returns

an NZCV flags producing node and sets OutCC to the flags that should be tested or returns SDValue() if transformation was not possible. Negate is true if we want this sub-tree being negated just by changing SETCC conditions.

Definition at line 1707 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, llvm::ISD::AND, assert(), canEmitConjunction(), changeFPCCToANDAArch64CC(), changeIntCCToAArch64CC(), emitComparison(), emitConditionalComparison(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::ISD::OR, llvm::ISD::SETCC, and std::swap().

Referenced by emitConjunction().

EmitVectorComparison()

static SDValue EmitVectorComparison ( SDValue  LHS, SDValue  RHS, AArch64CC::CondCode  CC, bool  NoNans, EVT  VT, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Definition at line 7714 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, assert(), changeIntCCToAArch64CC(), llvm::EVT::changeVectorElementTypeToInteger(), changeVectorFPCCToAArch64CC(), llvm::AArch64ISD::CMEQ, llvm::AArch64ISD::CMEQz, llvm::AArch64ISD::CMGE, llvm::AArch64ISD::CMGEz, llvm::AArch64ISD::CMGT, llvm::AArch64ISD::CMGTz, llvm::AArch64ISD::CMHI, llvm::AArch64ISD::CMHS, llvm::AArch64ISD::CMLEz, llvm::AArch64ISD::CMLTz, llvm::dyn_cast(), llvm::AArch64CC::EQ, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::AArch64ISD::FCMEQ, llvm::AArch64ISD::FCMEQz, llvm::AArch64ISD::FCMGE, llvm::AArch64ISD::FCMGEz, llvm::AArch64ISD::FCMGT, llvm::AArch64ISD::FCMGTz, llvm::AArch64ISD::FCMLEz, llvm::AArch64ISD::FCMLTz, llvm::ISD::FP_EXTEND, llvm::AArch64CC::GE, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::TargetLoweringBase::getTargetMachine(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::AArch64CC::LE, LLVM_FALLTHROUGH, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::bitc::NoNaNs, llvm::TargetOptions::NoNaNsFPMath, llvm::AArch64ISD::NOT, llvm::TargetMachine::Options, llvm::ISD::OR, llvm::SelectionDAG::ReplaceAllUsesWith(), resolveBuildVector(), llvm::MVT::v4f32, llvm::MVT::v4i16, and llvm::MVT::v4i32.

findEXTRHalf()

static bool findEXTRHalf ( SDValue  N, SDValue &  Src, uint32_t &  ShiftAmount, bool &  FromHi  )

static

An EXTR instruction is made up of two shifts, ORed together.

This helper searches for and classifies those shifts.

Definition at line 9227 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::ISD::SHL, and llvm::ISD::SRL.

Referenced by tryCombineToEXTR().

foldVectorXorShiftIntoCmp()

static SDValue foldVectorXorShiftIntoCmp ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.

Definition at line 8793 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::CMGEz, llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::AArch64Subtarget::hasNEON(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isVector(), and llvm::AArch64ISD::VASHR.

Referenced by performXorCombine().

GeneratePerfectShuffle()

static SDValue GeneratePerfectShuffle ( unsigned  PFEntry, SDValue  LHS, SDValue  RHS, SelectionDAG &  DAG, const SDLoc &  dl  )

static

GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle.

Definition at line 6406 of file AArch64ISelLowering.cpp.

References assert(), llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::AArch64ISD::EXT, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getConstant(), getExtFactor(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm_unreachable, PerfectShuffleTable, llvm::AArch64ISD::REV16, llvm::AArch64ISD::REV32, llvm::AArch64ISD::REV64, llvm::AArch64ISD::TRN1, llvm::AArch64ISD::TRN2, llvm::AArch64ISD::UZP1, llvm::AArch64ISD::UZP2, WidenVector(), llvm::AArch64ISD::ZIP1, and llvm::AArch64ISD::ZIP2.

Referenced by getDUPLANEOp().

GenerateTBL()

static SDValue GenerateTBL ( SDValue  Op, ArrayRef< int >  ShuffleMask, SelectionDAG &  DAG  )

static

Definition at line 6508 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_tbl1, llvm::Intrinsic::aarch64_neon_tbl2, llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorTemplateCommon< T >::data(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::SDNode::isUndef(), llvm::makeArrayRef(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, and llvm::MVT::v8i8.

Referenced by getDUPLANEOp().

getAArch64Cmp()

static SDValue getAArch64Cmp ( SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, SDValue &  AArch64cc, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Definition at line 1867 of file AArch64ISelLowering.cpp.

References C, changeIntCCToAArch64CC(), emitComparison(), emitConjunction(), getCmpOperandFoldingProfit(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasNUsesOfValue(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, INT64_MAX, isCMN(), isLegalArithImmed(), llvm::ConstantSDNode::isNullValue(), llvm::ConstantSDNode::isOne(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND_INREG, std::swap(), UINT64_MAX, and llvm::ISD::ZEXTLOAD.

Referenced by LowerXOR(), mayTailCallThisCC(), performSetccAddFolding(), and performXorCombine().

getAArch64XALUOOp()

static std::pair<SDValue, SDValue> getAArch64XALUOOp ( AArch64CC::CondCode &  CC, SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 1999 of file AArch64ISelLowering.cpp.

References llvm::MCID::Add, llvm::ISD::ADD, llvm::AArch64ISD::ADDS, assert(), llvm::MVT::f128, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::HS, llvm::MVT::i32, llvm::MVT::i64, llvm_unreachable, llvm::AArch64CC::LO, llvm::TargetLowering::makeLibCall(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::AArch64CC::NE, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::SADDO, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SSUBO, llvm::AArch64ISD::SUBS, llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::ISD::UMULO, llvm::ISD::USUBO, llvm::AArch64CC::VS, and llvm::ISD::ZERO_EXTEND.

Referenced by LowerXALUO(), LowerXOR(), and mayTailCallThisCC().

getCmpOperandFoldingProfit()

static unsigned getCmpOperandFoldingProfit ( SDValue  Op )

static

Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.

Definition at line 1833 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::BitmaskEnumDetail::Mask(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::ISD::SRL.

Referenced by getAArch64Cmp().

getDUPLANEOp()

static unsigned getDUPLANEOp ( EVT  EltType )

static

Definition at line 6570 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, Concat, llvm::ISD::CONCAT_VECTORS, llvm::AArch64ISD::DUP, llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::AArch64ISD::EXT, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, GeneratePerfectShuffle(), GenerateTBL(), llvm::SelectionDAG::getConstant(), getExtFactor(), llvm::ShuffleVectorSDNode::getMask(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::ShuffleVectorSDNode::getSplatIndex(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, isEXTMask(), isINSMask(), llvm::EVT::isInteger(), isREVMask(), isSingletonEXTMask(), llvm::ShuffleVectorSDNode::isSplat(), isTRN_v_undef_Mask(), isTRNMask(), llvm::SDNode::isUndef(), isUZP_v_undef_Mask(), isUZPMask(), isZIP_v_undef_Mask(), isZIPMask(), llvm_unreachable, PerfectShuffleTable, llvm::AArch64ISD::REV16, llvm::AArch64ISD::REV32, llvm::AArch64ISD::REV64, llvm::ISD::SCALAR_TO_VECTOR, std::swap(), llvm::AArch64ISD::TRN1, llvm::AArch64ISD::TRN2, tryFormConcatFromShuffle(), llvm::AArch64ISD::UZP1, llvm::AArch64ISD::UZP2, llvm::NVPTX::PTXLdStInstCode::V2, WidenVector(), llvm::AArch64ISD::ZIP1, and llvm::AArch64ISD::ZIP2.

Referenced by ConstantBuildVector().

getEstimate()

static SDValue getEstimate ( const AArch64Subtarget *  ST, unsigned  Opcode, SDValue  Operand, SelectionDAG &  DAG, int &  ExtraSteps  )

static

Definition at line 5464 of file AArch64ISelLowering.cpp.

References C, llvm::TargetLowering::C_Memory, llvm::TargetLowering::C_Other, llvm::TargetLowering::C_RegisterClass, llvm::TargetLowering::CW_Constant, llvm::TargetLowering::CW_Default, llvm::TargetLowering::CW_Invalid, llvm::TargetLowering::CW_Register, llvm::dyn_cast(), Enabled, llvm::TargetLoweringBase::Enabled, llvm::StringRef::equals_lower(), llvm::MVT::f32, llvm::MVT::f64, llvm::Failed(), llvm::ISD::FMUL, llvm::AArch64ISD::FRECPE, llvm::AArch64ISD::FRECPS, llvm::AArch64ISD::FRSQRTE, llvm::AArch64ISD::FRSQRTS, llvm::StringRef::getAsInteger(), llvm::SelectionDAG::getConstantFP(), llvm::TargetLowering::getConstraintType(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::TargetLowering::getRegForInlineAsmConstraint(), llvm::SelectionDAG::getRegister(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::AArch64TargetLowering::getSetCCResultType(), llvm::ConstantSDNode::getSExtValue(), llvm::TargetLowering::getSingleConstraintMatchWeight(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetBlockAddress(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::Value::getType(), llvm::SDValue::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::AArch64Subtarget::hasFPARMv8(), llvm::AArch64Subtarget::hasNEON(), llvm::MVT::i32, llvm::MVT::i64, info, llvm::EVT::isFloatingPoint(), llvm::Type::isFloatingPointTy(), llvm::AArch64_AM::isLogicalImmediate(), llvm::isNullConstant(), llvm::isUInt< 32 >(), llvm::EVT::isVector(), llvm::Type::isVectorTy(), llvm::TargetLowering::LowerAsmOperandForConstraint(), llvm::MVT::Other, llvm::ISD::SELECT, llvm::SDNodeFlags::setAllowReassociation(), llvm::ISD::SETEQ, Size, llvm::StringRef::size(), llvm::StringRef::slice(), TRI, llvm::AArch64Subtarget::useRSqrt(), llvm::MVT::v1f32, llvm::MVT::v1f64, llvm::MVT::v2f32, llvm::MVT::v2f64, llvm::MVT::v4f32, and llvm::ISD::VSELECT.

getExtensionTo64Bits()

static EVT getExtensionTo64Bits ( const EVT &  OrigVT )

static

Definition at line 2506 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::isSimple(), llvm_unreachable, llvm::MVT::SimpleTy, llvm::MVT::v2i16, llvm::MVT::v2i32, llvm::MVT::v2i8, llvm::MVT::v4i16, and llvm::MVT::v4i8.

Referenced by addRequiredExtensionForVectorMULL().

getExtFactor()

static unsigned getExtFactor ( SDValue &  V )

static

getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.

Definition at line 5900 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), and llvm::EVT::getVectorElementType().

Referenced by GeneratePerfectShuffle(), getDUPLANEOp(), and llvm::AArch64TargetLowering::ReconstructShuffle().

getIntrinsicID()

static unsigned getIntrinsicID ( const SDNode *  N )

static

Definition at line 7009 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::Intrinsic::not_intrinsic, and llvm::Intrinsic::num_intrinsics.

Referenced by llvm::CallBase::getCaller(), isBrevLdIntrinsic(), performExtendCombine(), performIntrinsicCombine(), and propagateMetadata().

getReductionSDNode()

static SDValue getReductionSDNode ( unsigned  Op, SDLoc  DL, SDValue  ScalarOp, SelectionDAG &  DAG  )

static

Definition at line 7873 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_fmaxnmv, llvm::Intrinsic::aarch64_neon_fminnmv, assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_CLR, llvm::AArch64ISD::CALL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getDataLayout(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getRegisterMask(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::AArch64RegisterInfo::getWindowsStackProbePreservedMask(), llvm::MVT::Glue, llvm::SDNodeFlags::hasNoNaNs(), llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::INTRINSIC_WO_CHAIN, llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::MVT::Other, llvm::ISD::SHL, Size, llvm::AArch64ISD::SMAXV, llvm::AArch64ISD::SMINV, llvm::ISD::SRL, llvm::ISD::SUB, llvm::AArch64ISD::UADDV, llvm::AArch64ISD::UMAXV, llvm::AArch64ISD::UMINV, llvm::AArch64RegisterInfo::UpdateCustomCallPreservedMask(), llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_FMAX, llvm::ISD::VECREDUCE_FMIN, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, and llvm::ISD::XOR.

getTestBitOperand()

static SDValue getTestBitOperand ( SDValue  Op, unsigned &  Bit, bool &  Invert, SelectionDAG &  DAG  )

static

Definition at line 10931 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::tgtok::Bit, C, llvm::dyn_cast(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by performTBZCombine().

getVShiftImm()

static bool getVShiftImm ( SDValue  Op, unsigned  ElementBits, int64_t &  Cnt  )

static

getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.

Definition at line 7626 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::APInt::getSExtValue(), and llvm::BuildVectorSDNode::isConstantSplat().

Referenced by isVShiftLImm(), and isVShiftRImm().

isAddSubSExt()

static bool isAddSubSExt ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 2600 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), isSignExtended(), and llvm::ISD::SUB.

Referenced by LowerMUL().

isAddSubZExt()

static bool isAddSubZExt ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 2611 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_get_fpcr, llvm::ISD::ADD, llvm::ISD::AND, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::INTRINSIC_WO_CHAIN, isZeroExtended(), llvm::ISD::SRL, llvm::ISD::SUB, and llvm::ISD::TRUNCATE.

Referenced by LowerMUL().

isAllConstantBuildVector()

static bool isAllConstantBuildVector ( const SDValue &  PotentialBVec, uint64_t &  ConstVal  )

static

Definition at line 6992 of file AArch64ISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::ConstantSDNode::getZExtValue().

Referenced by tryLowerToSLI().

isCMN()

static bool isCMN ( SDValue  Op, ISD::CondCode  CC  )

static

Definition at line 1494 of file AArch64ISelLowering.cpp.

References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::isNullConstant(), llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::ISD::SUB.

Referenced by emitComparison(), and getAArch64Cmp().

isConcatMask()

static bool isConcatMask ( ArrayRef< int >  Mask, EVT  VT, bool  SplitLHS  )

static

Definition at line 6355 of file AArch64ISelLowering.cpp.

References E, llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), and I.

Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal(), and tryFormConcatFromShuffle().

isEquivalentMaskless()

static bool isEquivalentMaskless ( unsigned  CC, unsigned  width, ISD::LoadExtType  ExtType, int  AddConstant, int  CompConstant  )

static

Definition at line 10715 of file AArch64ISelLowering.cpp.

References llvm::AArch64CC::AL, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::NV, llvm::AArch64CC::PL, llvm::ISD::SEXTLOAD, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.

Referenced by performCONDCombine().

isEssentiallyExtractSubvector()

static bool isEssentiallyExtractSubvector ( SDValue  N )

static

Definition at line 9629 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getOpcode(), and llvm::SDValue::getOperand().

Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().

isExtendedBUILD_VECTOR()

static bool isExtendedBUILD_VECTOR ( SDNode *  N, SelectionDAG &  DAG, bool  isSigned  )

static

Definition at line 2540 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, C, llvm::SDNode::getOpcode(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::isIntN(), llvm::isUIntN(), and llvm::SDNode::op_values().

Referenced by isSignExtended(), and isZeroExtended().

isEXTMask()

static bool isEXTMask ( ArrayRef< int >  M, EVT  VT, bool &  ReverseEXT, unsigned &  Imm  )

static

Definition at line 6160 of file AArch64ISelLowering.cpp.

References llvm::ArrayRef< T >::end(), llvm::find_if(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), and llvm::APInt::logBase2().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

isINSMask()

static bool isINSMask ( ArrayRef< int >  M, int  NumInputElements, bool &  DstIsLeft, int &  Anomaly  )

static

Definition at line 6316 of file AArch64ISelLowering.cpp.

References llvm::ArrayRef< T >::size().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

isLegalArithImmed()

static bool isLegalArithImmed ( uint64_t  C )

static

Definition at line 1476 of file AArch64ISelLowering.cpp.

References llvm::dbgs(), and LLVM_DEBUG.

Referenced by getAArch64Cmp().

isOverflowIntrOpRes()

static bool isOverflowIntrOpRes ( SDValue  Op )

static

Definition at line 2121 of file AArch64ISelLowering.cpp.

References llvm::SDValue::getOpcode(), llvm::SDValue::getResNo(), llvm::ISD::SADDO, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::ISD::UADDO, llvm::ISD::UMULO, and llvm::ISD::USUBO.

Referenced by LowerXOR(), and mayTailCallThisCC().

isREVMask()

static bool isREVMask ( ArrayRef< int >  M, EVT  VT, unsigned  BlockSize  )

static

isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.

(The order of the elements within each block of the vector is reversed.)

Definition at line 6201 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getScalarSizeInBits(), and llvm::EVT::getVectorNumElements().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

isSetCC()

static bool isSetCC ( SDValue  Op, SetCCInfoAndKind &  SetCCInfo  )

static

Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.

SetCCInfo is filled accordingly.

Postcondition

SetCCInfo is meanginfull only when this function returns true.

Returns

True when Op is a kind of SET_CC operation.

Definition at line 9670 of file AArch64ISelLowering.cpp.

References SetCCInfo::AArch64, GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, llvm::dyn_cast(), SetCCInfo::Generic, llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, llvm::ConstantSDNode::isOne(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, llvm::ISD::SETCC, and std::swap().

Referenced by isSetCCOrZExtSetCC().

isSetCCOrZExtSetCC()

static bool isSetCCOrZExtSetCC ( const SDValue &  Op, SetCCInfoAndKind &  Info  )

static

Definition at line 9713 of file AArch64ISelLowering.cpp.

References llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), Info, isSetCC(), and llvm::ISD::ZERO_EXTEND.

Referenced by performSetccAddFolding().

isSignExtended()

static bool isSignExtended ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 2590 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getOpcode(), isExtendedBUILD_VECTOR(), and llvm::ISD::SIGN_EXTEND.

Referenced by isAddSubSExt(), LowerMUL(), and performMulCombine().

isSingletonEXTMask()

static bool isSingletonEXTMask ( ArrayRef< int >  M, EVT  VT, unsigned &  Imm  )

static

Definition at line 6129 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by getDUPLANEOp().

isTRN_v_undef_Mask()

static bool isTRN_v_undef_Mask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.

Definition at line 6305 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

isTRNMask()

static bool isTRNMask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

Definition at line 6255 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

isUZP_v_undef_Mask()

static bool isUZP_v_undef_Mask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,

Definition at line 6286 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

isUZPMask()

static bool isUZPMask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

Definition at line 6242 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

isVShiftLImm()

static bool isVShiftLImm ( SDValue  Op, EVT  VT, bool  isLong, int64_t &  Cnt  )

static

isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.

That value must be in the range: 0 <= Value < ElementBits for a left shift; or 0 <= Value <= ElementBits for a long left shift.

Definition at line 7646 of file AArch64ISelLowering.cpp.

References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().

Referenced by isVShiftRImm().

isVShiftRImm()

static bool isVShiftRImm ( SDValue  Op, EVT  VT, bool  isNarrow, int64_t &  Cnt  )

static

isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.

The value must be in the range: 1 <= Value <= ElementBits for a right shift; or

Definition at line 7657 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_sshl, llvm::Intrinsic::aarch64_neon_ushl, assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), getVShiftImm(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isVector(), isVShiftLImm(), llvm_unreachable, llvm::AArch64ISD::NEG, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::AArch64ISD::VASHR, llvm::AArch64ISD::VLSHR, and llvm::AArch64ISD::VSHL.

isZeroExtended()

static bool isZeroExtended ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 2595 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getOpcode(), isExtendedBUILD_VECTOR(), and llvm::ISD::ZERO_EXTEND.

Referenced by isAddSubZExt(), LowerMUL(), and performMulCombine().

isZIP_v_undef_Mask()

static bool isZIP_v_undef_Mask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".

Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.

Definition at line 6269 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

isZIPMask()

static bool isZIPMask ( ArrayRef< int >  M, EVT  VT, unsigned &  WhichResult  )

static

Definition at line 6228 of file AArch64ISelLowering.cpp.

References llvm::EVT::getVectorNumElements().

Referenced by getDUPLANEOp(), and llvm::AArch64TargetLowering::isShuffleMaskLegal().

LowerADDC_ADDE_SUBC_SUBE()

static SDValue LowerADDC_ADDE_SUBC_SUBE ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 2208 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ADCS, llvm::ISD::ADDC, llvm::ISD::ADDE, llvm::AArch64ISD::ADDS, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::AArch64ISD::SBCS, llvm::ISD::SUBC, llvm::ISD::SUBE, and llvm::AArch64ISD::SUBS.

Referenced by llvm::AArch64TargetLowering::LowerOperation().

LowerBITCAST()

static SDValue LowerBITCAST ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 2491 of file AArch64ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, llvm::MVT::f16, llvm::MVT::f32, llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::MVT::i16, and llvm::MVT::i32.

Referenced by llvm::PPCTargetLowering::functionArgumentNeedsConsecutiveRegisters(), llvm::HexagonTargetLowering::isCtlzFast(), and llvm::AArch64TargetLowering::LowerOperation().

LowerMUL()

static SDValue LowerMUL ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 2642 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_abs, llvm::Intrinsic::aarch64_neon_smax, llvm::Intrinsic::aarch64_neon_smin, llvm::Intrinsic::aarch64_neon_umax, llvm::Intrinsic::aarch64_neon_umin, llvm::ISD::ABS, assert(), llvm::ISD::BITCAST, llvm::dyn_cast(), llvm::Intrinsic::eh_recoverfp, llvm::AArch64RegisterInfo::getBaseRegister(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::AArch64RegisterInfo::getFrameRegister(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::AArch64Subtarget::getRegisterInfo(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::AArch64RegisterInfo::hasBasePointer(), llvm::MVT::i64, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), isAddSubSExt(), isAddSubZExt(), llvm::EVT::isInteger(), isSignExtended(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), isZeroExtended(), llvm::Intrinsic::localaddress, Reg, llvm::report_fatal_error(), skipExtensionForVectorMULL(), llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::AArch64ISD::SMULL, std::swap(), llvm::AArch64ISD::THREAD_POINTER, llvm::Intrinsic::thread_pointer, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::AArch64ISD::UMULL, llvm::MVT::v1i64, and llvm::MVT::v2i64.

Referenced by llvm::PPCTargetLowering::functionArgumentNeedsConsecutiveRegisters(), and llvm::AArch64TargetLowering::LowerOperation().

LowerPREFETCH()

static SDValue LowerPREFETCH ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 2275 of file AArch64ISelLowering.cpp.

References assert(), llvm::MVT::f128, first, llvm::SelectionDAG::getConstant(), llvm::RTLIB::getFPEXT(), llvm::RTLIB::getFPROUND(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::MVT::i32, llvm::TargetLowering::makeLibCall(), llvm::MVT::Other, and llvm::AArch64ISD::PREFETCH.

Referenced by llvm::HexagonTargetLowering::isCtlzFast(), and llvm::AArch64TargetLowering::LowerOperation().

LowerTruncateVectorStore()

static SDValue LowerTruncateVectorStore ( SDLoc  DL, StoreSDNode *  ST, EVT  VT, EVT  MemVT, SelectionDAG &  DAG  )

static

Definition at line 2778 of file AArch64ISelLowering.cpp.

References llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, llvm::AArch64TargetLowering::allowsMisalignedMemoryAccesses(), assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSize(), llvm::SelectionDAG::getUNDEF(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::StoreSDNode::isTruncatingStore(), llvm::EVT::isVector(), llvm::TargetLowering::scalarizeVectorStore(), llvm::ISD::TRUNCATE, llvm::RegState::Undef, llvm::MVT::v2i32, llvm::MVT::v4i16, llvm::MVT::v4i8, llvm::MVT::v8i16, and llvm::MVT::v8i8.

LowerVectorFP_TO_INT()

static SDValue LowerVectorFP_TO_INT ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 2329 of file AArch64ISelLowering.cpp.

References llvm::EVT::changeVectorElementTypeToInteger(), llvm::MipsISD::Ext, llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f32, first, llvm::ISD::FP_EXTEND, llvm::ISD::FP_TO_SINT, llvm::MVT::getFloatingPointVT(), llvm::RTLIB::getFPTOSINT(), llvm::RTLIB::getFPTOUINT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::AArch64Subtarget::hasFullFP16(), llvm::EVT::isVector(), llvm::TargetLowering::makeLibCall(), llvm::SDNode::op_begin(), llvm::SDNode::op_end(), and llvm::ISD::TRUNCATE.

LowerVectorINT_TO_FP()

static SDValue LowerVectorINT_TO_FP ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 2396 of file AArch64ISelLowering.cpp.

References Arg, llvm::AMDGPU::HSAMD::Kernel::Key::Args, Callee, llvm::EVT::changeVectorElementTypeToInteger(), llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::CallingConv::Fast, llvm::ISD::FP_ROUND, llvm::StructType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExternalSymbol(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::TargetLoweringBase::getLibcallName(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarSizeInBits(), llvm::RTLIB::getSINTTOFP(), llvm::EVT::getSizeInBits(), llvm::EVT::getTypeForEVT(), llvm::RTLIB::getUINTTOFP(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::AArch64Subtarget::hasFullFP16(), llvm::MVT::i128, llvm::tgtok::In, llvm::EVT::isVector(), llvm::TargetLowering::LowerCallTo(), llvm::TargetLowering::CallLoweringInfo::setChain(), llvm::TargetLowering::CallLoweringInfo::setDebugLoc(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, and llvm::ISD::ZERO_EXTEND.

LowerXALUO()

static SDValue LowerXALUO ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 2244 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::CSEL, getAArch64XALUOOp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::TargetLoweringBase::isTypeLegal(), and llvm::ISD::MERGE_VALUES.

Referenced by llvm::AArch64TargetLowering::LowerOperation().

LowerXOR()

static SDValue LowerXOR ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 2128 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::CSEL, llvm::dyn_cast(), getAArch64Cmp(), getAArch64XALUOOp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, llvm::ConstantSDNode::isAllOnesValue(), llvm::ConstantSDNode::isNullValue(), llvm::isOneConstant(), isOverflowIntrOpRes(), llvm::TargetLoweringBase::isTypeLegal(), Other, llvm::ISD::SELECT_CC, std::swap(), and llvm::ISD::XOR.

Referenced by llvm::AArch64TargetLowering::LowerOperation().

mayTailCallThisCC()

static bool mayTailCallThisCC ( CallingConv::ID  CC )

static

Return true if we might ever do TCO for calls with this calling convention.

Definition at line 3367 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_uaddlp, llvm::Intrinsic::aarch64_neon_uaddlv, llvm::ISD::ADD, llvm::MachineFunction::addLiveIn(), llvm::AArch64ISD::ADDlow, llvm::AArch64ISD::ADR, llvm::AArch64ISD::ADRP, llvm::CCValAssign::AExt, llvm::AArch64CC::AL, llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, llvm::alignTo(), llvm::CCState::AnalyzeCallOperands(), llvm::CCState::AnalyzeReturn(), llvm::ISD::AND, llvm::ISD::ANY_EXTEND, Arg, llvm::Function::arg_begin(), llvm::Function::arg_end(), assert(), llvm::CCValAssign::BCvt, llvm::AArch64ISD::BIT, llvm::ISD::BITCAST, llvm::EVT::bitsGT(), llvm::EVT::bitsLT(), llvm::AArch64ISD::BRCOND, llvm::ISD::BRIND, llvm::CallingConv::C, C, llvm::AArch64ISD::CALL, Callee, canGuaranteeTCO(), llvm::StringSwitch< T, R >::Case(), llvm::AArch64ISD::CBNZ, llvm::AArch64ISD::CBZ, llvm::AArch64TargetLowering::CCAssignFnForCall(), changeFPCCToAArch64CC(), changeIntCCToAArch64CC(), llvm::CCState::CheckReturn(), llvm::AArch64Subtarget::classifyGlobalFunctionReference(), llvm::AArch64Subtarget::ClassifyGlobalReference(), contains(), Context, llvm::HexagonISD::CP, llvm::MachineFrameInfo::CreateFixedObject(), llvm::AArch64ISD::CSEL, llvm::AArch64ISD::CSINC, llvm::AArch64ISD::CSINV, llvm::AArch64ISD::CSNEG, llvm::ISD::CTPOP, llvm::dbgs(), llvm::StringSwitch< T, R >::Default(), llvm::Depth, llvm::dyn_cast(), emitComparison(), llvm::AArch64RegisterInfo::emitReservedArgRegCallError(), llvm::SmallVectorBase::empty(), EnableAArch64ELFLocalDynamicTLSGeneration, llvm::AArch64CC::EQ, F(), llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f32, llvm::MVT::f64, llvm::CallingConv::Fast, llvm::ISD::FNEG, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::CCValAssign::FPExt, llvm::CCValAssign::Full, G, llvm::AArch64CC::GE, llvm::TLSModel::GeneralDynamic, getAArch64Cmp(), getAArch64XALUOOp(), llvm::ConstantPoolSDNode::getAlignment(), llvm::SelectionDAG::getBitcast(), llvm::BlockAddressSDNode::getBlockAddress(), llvm::AArch64FunctionInfo::getBytesInStackArgArea(), llvm::ISD::ArgFlagsTy::getByValAlign(), llvm::ISD::ArgFlagsTy::getByValSize(), llvm::AArch64RegisterInfo::getCalleeSavedRegsViaCopy(), llvm::Function::getCallingConv(), llvm::AArch64RegisterInfo::getCallPreservedMask(), llvm::SelectionDAG::getCALLSEQ_END(), llvm::SelectionDAG::getCALLSEQ_START(), llvm::TargetMachine::getCodeModel(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::ConstantPoolSDNode::getConstVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachinePointerInfo::getFixedStack(), llvm::MVT::getFloatingPointVT(), llvm::AArch64FunctionInfo::getForwardedMustTailRegParms(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachinePointerInfo::getGOT(), llvm::JumpTableSDNode::getIndex(), llvm::MachineFunction::getInfo(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getLoad(), llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocMemOffset(), llvm::CCValAssign::getLocReg(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMemcpy(), llvm::SelectionDAG::getMergeValues(), llvm::CCState::getNextStackOffset(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::MachineFrameInfo::getObjectOffset(), llvm::MachineFrameInfo::getObjectSize(), llvm::getOffset(), llvm::GlobalAddressSDNode::getOffset(), llvm::ConstantPoolSDNode::getOffset(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::MachineFunction::getRegInfo(), llvm::SelectionDAG::getRegister(), llvm::AArch64Subtarget::getRegisterInfo(), llvm::SelectionDAG::getRegisterMask(), llvm::EVT::getScalarSizeInBits(), llvm::ISD::getSetCCInverse(), llvm::ConstantSDNode::getSExtValue(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MachinePointerInfo::getStack(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTarget(), llvm::MachineFunction::getTarget(), llvm::SelectionDAG::getTargetBlockAddress(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetConstantPool(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SelectionDAG::getTargetInsertSubreg(), llvm::SelectionDAG::getTargetJumpTable(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTargetMachine(), llvm::TargetMachine::getTargetTriple(), llvm::AArch64RegisterInfo::getThisReturnPreservedMask(), llvm::AArch64RegisterInfo::getTLSCallPreservedMask(), llvm::TargetMachine::getTLSModel(), llvm::DataLayout::getTypeAllocSize(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::CCValAssign::getValVT(), llvm::AArch64FunctionInfo::getVarArgsFPRIndex(), llvm::AArch64FunctionInfo::getVarArgsFPRSize(), llvm::AArch64FunctionInfo::getVarArgsGPRIndex(), llvm::AArch64FunctionInfo::getVarArgsGPRSize(), llvm::AArch64FunctionInfo::getVarArgsStackIndex(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::Glue, llvm::TargetOptions::GuaranteedTailCallOpt, llvm::AArch64Subtarget::hasCustomCallingConv(), llvm::GlobalValue::hasExternalWeakLinkage(), llvm::Function::hasFnAttribute(), llvm::AArch64Subtarget::hasFullFP16(), llvm::AArch64Subtarget::hasNEON(), llvm::MipsISD::Hi, I, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::AArch64FunctionInfo::incNumLocalDynamicTLSAccesses(), llvm::TLSModel::InitialExec, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::is64BitVector(), llvm::isAllOnesConstant(), llvm::ConstantSDNode::isAllOnesValue(), llvm::AArch64RegisterInfo::isAnyArgRegReserved(), llvm::ISD::ArgFlagsTy::isByVal(), llvm::AArch64Subtarget::isCallingConvWin64(), llvm::EVT::isFloatingPoint(), llvm::ISD::ArgFlagsTy::isInConsecutiveRegs(), llvm::EVT::isInteger(), llvm::AArch64Subtarget::isLittleEndian(), llvm::CCValAssign::isMemLoc(), llvm::isNullConstant(), llvm::ConstantSDNode::isNullValue(), llvm::ConstantSDNode::isOne(), llvm::isOneConstant(), llvm::Triple::isOSBinFormatELF(), llvm::Triple::isOSBinFormatMachO(), llvm::Triple::isOSWindows(), isOverflowIntrOpRes(), llvm::isPowerOf2_64(), llvm::CCValAssign::isRegLoc(), llvm::ISD::ArgFlagsTy::isReturned(), llvm::EVT::isSimple(), llvm::ISD::ArgFlagsTy::isSwiftSelf(), llvm::AArch64Subtarget::isTargetCOFF(), llvm::AArch64Subtarget::isTargetDarwin(), llvm::AArch64Subtarget::isTargetELF(), llvm::AArch64Subtarget::isTargetMachO(), llvm::AArch64Subtarget::isTargetWindows(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::AArch64Subtarget::isXRegisterReserved(), llvm::ConstantFPSDNode::isZero(), llvm::HexagonISD::JT, llvm::CodeModel::Large, LLVM_DEBUG, llvm_unreachable, llvm::MipsISD::Lo, llvm::AArch64ISD::LOADgot, llvm::TLSModel::LocalDynamic, llvm::TLSModel::LocalExec, llvm::Log2_64(), llvm::TargetLowering::LowerToTLSEmulatedModel(), llvm::BitmaskEnumDetail::Mask(), llvm::AArch64II::MO_COFFSTUB, llvm::AArch64II::MO_DLLIMPORT, llvm::AArch64II::MO_G0, llvm::AArch64II::MO_G1, llvm::AArch64II::MO_G2, llvm::AArch64II::MO_G3, llvm::AArch64II::MO_GOT, llvm::AArch64II::MO_HI12, llvm::AArch64II::MO_NC, llvm::AArch64II::MO_NO_FLAG, llvm::AArch64II::MO_PAGE, llvm::AArch64II::MO_PAGEOFF, llvm::AArch64II::MO_TLS, llvm::MachineMemOperand::MODereferenceable, llvm::MachineMemOperand::MOInvariant, llvm::MachineMemOperand::MONonTemporal, MRI, N, llvm::AArch64CC::NE, llvm::Attribute::NoImplicitFloat, llvm::TargetMachine::Options, llvm::ISD::OR, llvm::MVT::Other, llvm::TargetLowering::parametersInCSRMatch(), llvm::CallingConv::PreserveMost, llvm::SmallVectorTemplateBase< T >::push_back(), Reg, llvm::report_fatal_error(), llvm::CCState::resultsCompatible(), llvm::AArch64ISD::RET_FLAG, llvm::MachineFrameInfo::setAdjustsStack(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::MachineFrameInfo::setHasTailCall(), llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETONE, llvm::MachineFrameInfo::setReturnAddressIsTaken(), llvm::ISD::SETUEQ, llvm::ISD::SETUNE, llvm::CCValAssign::SExt, llvm::ISD::SHL, llvm::ISD::SHL_PARTS, llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase::size(), llvm::TargetLowering::softenSetCCOperands(), llvm::Attribute::SpeculativeLoadHardening, llvm::ISD::SRA, llvm::ISD::SRA_PARTS, llvm::ISD::SRL, llvm::ISD::SRL_PARTS, llvm::SPII::Store, llvm::ISD::SUB, std::swap(), llvm::CallingConv::Swift, llvm::AArch64ISD::TBNZ, llvm::AArch64ISD::TBZ, llvm::AArch64ISD::TC_RETURN, llvm::AArch64ISD::THREAD_POINTER, llvm::CodeModel::Tiny, llvm::AArch64ISD::TLSDESC_CALLSEQ, llvm::ISD::TokenFactor, TRI, llvm::ISD::TRUNCATE, llvm::TargetOptions::UnsafeFPMath, llvm::AArch64RegisterInfo::UpdateCustomCallPreservedMask(), llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::TargetMachine::useEmulatedTLS(), llvm::MVT::v16i8, llvm::MVT::v1i64, llvm::MVT::v2f32, llvm::MVT::v2f64, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4f16, llvm::MVT::v4f32, llvm::MVT::v4i16, llvm::MVT::v4i32, llvm::MVT::v8f16, llvm::MVT::v8i16, llvm::MVT::v8i8, llvm::CallingConv::WebKit_JS, llvm::AArch64ISD::WrapperLarge, llvm::ISD::XOR, llvm::ISD::ZERO_EXTEND, and llvm::CCValAssign::ZExt.

memOpAlign()

static bool memOpAlign ( unsigned  DstAlign, unsigned  SrcAlign, unsigned  AlignCheck  )

static

Definition at line 8579 of file AArch64ISelLowering.cpp.

Referenced by llvm::AArch64TargetLowering::getOptimalMemOpType().

NarrowVector()

static SDValue NarrowVector ( SDValue  V128Reg, SelectionDAG &  DAG  )

static

NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 register class.

Definition at line 5907 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), and llvm::MVT::getVectorVT().

NormalizeBuildVector()

static SDValue NormalizeBuildVector ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 7133 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getOpcode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i32, llvm::EVT::isFloatingPoint(), llvm::SDNode::ops(), and llvm::SmallVectorTemplateBase< T >::push_back().

Referenced by ConstantBuildVector().

optimizeLogicalImm()

static bool optimizeLogicalImm ( SDValue  Op, unsigned  Size, uint64_t  Imm, const APInt &  Demanded, TargetLowering::TargetLoweringOpt &  TLO, unsigned  NewOpc  )

static

Definition at line 851 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::TargetLoweringOpt::CombineTo(), llvm::TargetLowering::TargetLoweringOpt::DAG, llvm::AArch64_AM::encodeLogicalImmediate(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::APInt::getZExtValue(), llvm::MipsISD::Hi, llvm::AArch64_AM::isLogicalImmediate(), llvm::isShiftedMask_64(), llvm::BitmaskEnumDetail::Mask(), and Size.

Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().

performAddSubLongCombine()

static SDValue performAddSubLongCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 9776 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractSubvector(), performSetccAddFolding(), llvm::ISD::SIGN_EXTEND, tryExtendDUPToExtractHigh(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performBitcastCombine()

static SDValue performBitcastCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 9382 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::dbgs(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SDNode::getMachineOpcode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i32, llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::SDNode::isMachineOpcode(), llvm::EVT::isVector(), LLVM_DEBUG, llvm::Sched::Source, and SubReg.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performBRCONDCombine()

static SDValue performBRCONDCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 10865 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::ADDS, assert(), llvm::ISD::BR, llvm::AArch64ISD::CBNZ, llvm::AArch64ISD::CBZ, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::AArch64CC::EQ, llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::Function::hasFnAttribute(), llvm::SDNode::hasNUsesOfValue(), llvm::MVT::i32, llvm::MVT::i64, llvm::isNullConstant(), llvm::AArch64CC::NE, llvm::MVT::Other, performCONDCombine(), llvm::ISD::SHL, llvm::Attribute::SpeculativeLoadHardening, llvm::ISD::SRA, llvm::ISD::SRL, llvm::AArch64ISD::SUBS, and std::swap().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performConcatVectorsCombine()

static SDValue performConcatVectorsCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 9450 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::dbgs(), llvm::AArch64ISD::DUPLANE64, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::MVT::isVector(), LLVM_DEBUG, llvm::BitmaskEnumDetail::Mask(), llvm::ISD::TRUNCATE, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, and WidenVector().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performCONDCombine()

static SDValue performCONDCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG, unsigned  CCIndex, unsigned  CmpIndex  )

static

Definition at line 10791 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, checkValueWidth(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), isEquivalentMaskless(), llvm::SelectionDAG::ReplaceAllUsesWith(), and llvm::AArch64ISD::SUBS.

Referenced by performBRCONDCombine(), and llvm::AArch64TargetLowering::PerformDAGCombine().

performExtendCombine()

static SDValue performExtendCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 9996 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_sabd, llvm::Intrinsic::aarch64_neon_uabd, assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getIntegerVT(), getIntrinsicID(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::MipsISD::Hi, llvm::MVT::i64, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::MipsISD::Lo, tryCombineLongOpWithDup(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFDivCombine()

static SDValue performFDivCombine ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Fold a floating-point divide by power of two into fixed-point to floating-point conversion.

Definition at line 9158 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_vcvtfxs2fp, llvm::Intrinsic::aarch64_neon_vcvtfxu2fp, C, llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::AArch64Subtarget::hasNEON(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, llvm::ISD::UINT_TO_FP, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v4i64, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performFpToIntCombine()

static SDValue performFpToIntCombine ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.

Definition at line 9086 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_vcvtfp2fxs, llvm::Intrinsic::aarch64_neon_vcvtfp2fxu, assert(), llvm::tgtok::Bits, C, llvm::ISD::FMUL, llvm::ISD::FP_TO_SINT, llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::AArch64Subtarget::hasNEON(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::ISD::TRUNCATE, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i32, and llvm::MVT::v4i64.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performGlobalAddressCombine()

static SDValue performGlobalAddressCombine ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget, const TargetMachine &  TM  )

static

Definition at line 11129 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, C, llvm::AArch64Subtarget::ClassifyGlobalReference(), llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::Module::getDataLayout(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::GlobalValue::getParent(), llvm::DataLayout::getTypeAllocSize(), llvm::GlobalValue::getValueType(), llvm::MVT::i64, llvm::Type::isSized(), llvm::AArch64II::MO_NO_FLAG, N, llvm::ISD::SUB, and llvm::SDNode::uses().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performIntegerAbsCombine()

static SDValue performIntegerAbsCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 8817 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::AArch64ISD::CSEL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::EVT::isInteger(), llvm::AArch64CC::PL, llvm::ISD::SRA, llvm::ISD::SUB, llvm::AArch64ISD::SUBS, and llvm::ISD::XOR.

Referenced by performXorCombine().

performIntrinsicCombine()

static SDValue performIntrinsicCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 9940 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_crc32b, llvm::Intrinsic::aarch64_crc32cb, llvm::Intrinsic::aarch64_crc32ch, llvm::Intrinsic::aarch64_crc32h, llvm::Intrinsic::aarch64_neon_fmax, llvm::Intrinsic::aarch64_neon_fmaxnm, llvm::Intrinsic::aarch64_neon_fmin, llvm::Intrinsic::aarch64_neon_fminnm, llvm::Intrinsic::aarch64_neon_pmull, llvm::Intrinsic::aarch64_neon_saddv, llvm::Intrinsic::aarch64_neon_smaxv, llvm::Intrinsic::aarch64_neon_sminv, llvm::Intrinsic::aarch64_neon_smull, llvm::Intrinsic::aarch64_neon_sqdmull, llvm::Intrinsic::aarch64_neon_sqshl, llvm::Intrinsic::aarch64_neon_sqshlu, llvm::Intrinsic::aarch64_neon_srshl, llvm::Intrinsic::aarch64_neon_uaddv, llvm::Intrinsic::aarch64_neon_umaxv, llvm::Intrinsic::aarch64_neon_uminv, llvm::Intrinsic::aarch64_neon_umull, llvm::Intrinsic::aarch64_neon_uqshl, llvm::Intrinsic::aarch64_neon_urshl, llvm::Intrinsic::aarch64_neon_vcvtfxs2fp, llvm::Intrinsic::aarch64_neon_vcvtfxu2fp, combineAcrossLanesIntrinsic(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::FMAXIMUM, llvm::ISD::FMAXNUM, llvm::ISD::FMINIMUM, llvm::ISD::FMINNUM, getIntrinsicID(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::AArch64ISD::SADDV, llvm::AArch64ISD::SMAXV, llvm::AArch64ISD::SMINV, tryCombineCRC32(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineShiftImm(), llvm::AArch64ISD::UADDV, llvm::AArch64ISD::UMAXV, and llvm::AArch64ISD::UMINV.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performIntToFpCombine()

static SDValue performIntToFpCombine ( SDNode *  N, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 9045 of file AArch64ISelLowering.cpp.

References llvm::MVT::f32, llvm::MVT::f64, llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::AArch64Subtarget::hasNEON(), llvm::SDValue::hasOneUse(), llvm::ISD::isNormalLoad(), isVolatile(), llvm::SPII::Load, performVectorCompareAndMaskUnaryOpCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SINT_TO_FP, llvm::AArch64ISD::SITOF, and llvm::AArch64ISD::UITOF.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performMulCombine()

static SDValue performMulCombine ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 8899 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::APInt::ashr(), assert(), C, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isPowerOf2(), isSignExtended(), isZeroExtended(), llvm::APInt::logBase2(), llvm::ISD::SHL, llvm::ISD::SUB, and llvm::SDNode::use_begin().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performNEONPostLDSTCombine()

static SDValue performNEONPostLDSTCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.

Definition at line 10473 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_ld1x2, llvm::Intrinsic::aarch64_neon_ld1x3, llvm::Intrinsic::aarch64_neon_ld1x4, llvm::Intrinsic::aarch64_neon_ld2, llvm::Intrinsic::aarch64_neon_ld2lane, llvm::Intrinsic::aarch64_neon_ld2r, llvm::Intrinsic::aarch64_neon_ld3, llvm::Intrinsic::aarch64_neon_ld3lane, llvm::Intrinsic::aarch64_neon_ld3r, llvm::Intrinsic::aarch64_neon_ld4, llvm::Intrinsic::aarch64_neon_ld4lane, llvm::Intrinsic::aarch64_neon_ld4r, llvm::Intrinsic::aarch64_neon_st1x2, llvm::Intrinsic::aarch64_neon_st1x3, llvm::Intrinsic::aarch64_neon_st1x4, llvm::Intrinsic::aarch64_neon_st2, llvm::Intrinsic::aarch64_neon_st2lane, llvm::Intrinsic::aarch64_neon_st3, llvm::Intrinsic::aarch64_neon_st3lane, llvm::Intrinsic::aarch64_neon_st4, llvm::Intrinsic::aarch64_neon_st4lane, llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::MVT::i64, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::AArch64ISD::LD1x2post, llvm::AArch64ISD::LD1x3post, llvm::AArch64ISD::LD1x4post, llvm::AArch64ISD::LD2DUPpost, llvm::AArch64ISD::LD2LANEpost, llvm::AArch64ISD::LD2post, llvm::AArch64ISD::LD3DUPpost, llvm::AArch64ISD::LD3LANEpost, llvm::AArch64ISD::LD3post, llvm::AArch64ISD::LD4DUPpost, llvm::AArch64ISD::LD4LANEpost, llvm::AArch64ISD::LD4post, llvm_unreachable, llvm::makeArrayRef(), N, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::AArch64ISD::ST1x2post, llvm::AArch64ISD::ST1x3post, llvm::AArch64ISD::ST1x4post, llvm::AArch64ISD::ST2LANEpost, llvm::AArch64ISD::ST2post, llvm::AArch64ISD::ST3LANEpost, llvm::AArch64ISD::ST3post, llvm::AArch64ISD::ST4LANEpost, llvm::AArch64ISD::ST4post, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performNVCASTCombine()

static SDValue performNVCASTCombine ( SDNode *  N )

static

Get rid of unnecessary NVCASTs (that don't change the type).

Definition at line 11119 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), and llvm::SDNode::getValueType().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performORCombine()

static SDValue performORCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 9336 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::TargetLoweringBase::isTypeLegal(), tryCombineToBSL(), and tryCombineToEXTR().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performPostLD1Combine()

static SDValue performPostLD1Combine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, bool  IsLaneOp  )

static

Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.

Definition at line 10337 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::MVT::i64, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ARM_MB::LD, llvm::AArch64ISD::LD1DUPpost, llvm::AArch64ISD::LD1LANEpost, llvm::ISD::LOAD, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SDNode::use_begin(), and llvm::SDNode::use_end().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSelectCombine()

static SDValue performSelectCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar.

This replaces any scalar setcc in the above pattern with a vector one followed by a DUP shuffle on the result.

Definition at line 11058 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::i1, llvm::MVT::i32, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::BitmaskEnumDetail::Mask(), llvm::ISD::SCALAR_TO_VECTOR, and llvm::ISD::SETCC.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSetccAddFolding()

static SDValue performSetccAddFolding ( SDNode *  Op, SelectionDAG &  DAG  )

static

Definition at line 9726 of file AArch64ISelLowering.cpp.

References SetCCInfo::AArch64, llvm::ISD::ADD, assert(), GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, SetCCInfo::Generic, getAArch64Cmp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, isSetCCOrZExtSetCC(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, and std::swap().

Referenced by performAddSubLongCombine().

performSRLCombine()

static SDValue performSRLCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 9354 of file AArch64ISelLowering.cpp.

References llvm::ISD::BSWAP, C, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, llvm::SelectionDAG::MaskedValueIsZero(), and llvm::ISD::ROTR.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performSTORECombine()

static SDValue performSTORECombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 10456 of file AArch64ISelLowering.cpp.

References llvm::SDNode::getOperand(), performTBISimplification(), Split(), splitStores(), and llvm::AArch64Subtarget::supportsAddressTopByteIgnored().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performTBISimplification()

static bool performTBISimplification ( SDValue  Addr, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Simplify Addr given that the top byte of it is ignored by HW during address translation.

Definition at line 10441 of file AArch64ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), and llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps().

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), and performSTORECombine().

performTBZCombine()

static SDValue performTBZCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 10998 of file AArch64ISelLowering.cpp.

References assert(), llvm::tgtok::Bit, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), getTestBitOperand(), llvm::MVT::i64, llvm::MVT::Other, llvm::AArch64ISD::TBNZ, and llvm::AArch64ISD::TBZ.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performVectorCompareAndMaskUnaryOpCombine()

static SDValue performVectorCompareAndMaskUnaryOpCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 8999 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BITCAST, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::isVector(), and llvm::ISD::SETCC.

Referenced by performIntToFpCombine().

performVSelectCombine()

static SDValue performVSelectCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 11029 of file AArch64ISelLowering.cpp.

References llvm::EVT::changeVectorElementTypeToInteger(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i1, llvm::ISD::SETCC, and llvm::ISD::VSELECT.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

performXorCombine()

static SDValue performXorCombine ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const AArch64Subtarget *  Subtarget  )

static

Definition at line 8844 of file AArch64ISelLowering.cpp.

References llvm::MCID::Add, llvm::ISD::ADD, llvm::APInt::countTrailingZeros(), llvm::AArch64ISD::CSEL, foldVectorXorShiftIntoCmp(), getAArch64Cmp(), llvm::Function::getAttributes(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::AArch64TargetLowering::isIntDivCheap(), llvm::APInt::isNonNegative(), llvm::APInt::isPowerOf2(), performIntegerAbsCombine(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SETLT, llvm::ISD::SRA, and llvm::ISD::SUB.

Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().

ReplaceBITCASTResults()

static void ReplaceBITCASTResults ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG  )

static

Definition at line 11389 of file AArch64ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::MVT::f16, llvm::MVT::f32, llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i16, llvm::MVT::i32, llvm::SmallVectorTemplateBase< T >::push_back(), and llvm::ISD::TRUNCATE.

Referenced by ReplaceCMP_SWAP_128Results().

ReplaceCMP_SWAP_128Results()

static void ReplaceCMP_SWAP_128Results ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 11447 of file AArch64ISelLowering.cpp.

References llvm::Acquire, llvm::AcquireRelease, llvm::ISD::ADD, assert(), llvm::ISD::ATOMIC_CMP_SWAP, llvm::ISD::BITCAST, createGPRPairNode(), llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_UINT, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::MachineMemOperand::getOrdering(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::AArch64Subtarget::hasLSE(), llvm::MVT::i128, llvm::MVT::i32, llvm::MVT::i64, llvm::DataLayout::isBigEndian(), llvm_unreachable, llvm::Monotonic, N, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::Release, ReplaceBITCASTResults(), ReplaceReductionResults(), Results, llvm::AArch64ISD::SADDV, llvm::SequentiallyConsistent, llvm::SelectionDAG::setNodeMemRefs(), llvm::ISD::SMAX, llvm::AArch64ISD::SMAXV, llvm::ISD::SMIN, llvm::AArch64ISD::SMINV, splitInt128(), std::swap(), llvm::AArch64ISD::UADDV, llvm::ISD::UMAX, llvm::AArch64ISD::UMAXV, llvm::ISD::UMIN, llvm::AArch64ISD::UMINV, llvm::MVT::Untyped, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, and llvm::ISD::VECREDUCE_UMIN.

ReplaceReductionResults()

static void ReplaceReductionResults ( SDNode *  N, SmallVectorImpl< SDValue > &  Results, SelectionDAG &  DAG, unsigned  InterOp, unsigned  AcrossOp  )

static

Definition at line 11406 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::SDNode::getValueType(), llvm::MipsISD::Hi, llvm::MipsISD::Lo, llvm::SmallVectorTemplateBase< T >::push_back(), and llvm::SelectionDAG::SplitVectorOperand().

Referenced by ReplaceCMP_SWAP_128Results().

replaceSplatVectorStore()

static SDValue replaceSplatVectorStore ( SelectionDAG &  DAG, StoreSDNode &  St  )

static

Replace a splat of a scalar to a vector store by scalar stores of the scalar value.

The load store optimizer pass will merge them to store pair stores. This has better performance than a splat of the scalar followed by a split vector store. Even if the stores are not merged it is four stores vs a dup, followed by an ext.b and two stores.

Definition at line 10212 of file AArch64ISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), llvm::StoreSDNode::isTruncatingStore(), and splitStoreSplat().

Referenced by splitStores().

replaceZeroVectorStore()

static SDValue replaceZeroVectorStore ( SelectionDAG &  DAG, StoreSDNode &  St  )

static

Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.

The load store optimizer pass will merge them to store pair stores. This should be better than a movi to create the vector zero followed by a vector store if the zero constant is not re-used, since one instructions and one register live range will be removed.

For example, the final generated code should be:

stp xzr, xzr, [x0]

instead of:

movi v0.2d, #0 str q0, [x0]

Definition at line 10149 of file AArch64ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::StoreSDNode::getBasePtr(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), I, llvm::MVT::i32, llvm::MVT::i64, llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::isNullConstant(), llvm::isNullFPConstant(), llvm::StoreSDNode::isTruncatingStore(), and splitStoreSplat().

Referenced by splitStores().

resolveBuildVector()

static bool resolveBuildVector ( BuildVectorSDNode *  BVN, APInt &  CnstBits, APInt &  UndefBits  )

static

Definition at line 6735 of file AArch64ISelLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstantSplat(), and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector(), EmitVectorComparison(), tryAdvSIMDModImmFP(), and tryLowerToSLI().

skipExtensionForVectorMULL()

static SDValue skipExtensionForVectorMULL ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 2566 of file AArch64ISelLowering.cpp.

References addRequiredExtensionForVectorMULL(), assert(), llvm::ISD::BUILD_VECTOR, C, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::MVT::getVectorVT(), llvm::MVT::i32, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by LowerMUL().

splitInt128()

static std::pair<SDValue, SDValue> splitInt128 ( SDValue  N, SelectionDAG &  DAG  )

static

Definition at line 11420 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MipsISD::Hi, llvm::MVT::i128, llvm::MVT::i64, llvm::MipsISD::Lo, llvm::ISD::SRL, and llvm::ISD::TRUNCATE.

Referenced by ReplaceCMP_SWAP_128Results().

splitStores()

static SDValue splitStores ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG, const AArch64Subtarget *  Subtarget  )

static

Definition at line 10265 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MemSDNode::getAlignment(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::i64, llvm::LSBaseSDNode::isIndexed(), llvm::AArch64Subtarget::isMisaligned128StoreSlow(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), N, llvm::Function::optForMinSize(), replaceSplatVectorStore(), replaceZeroVectorStore(), and llvm::MVT::v2i64.

Referenced by performSTORECombine().

splitStoreSplat()

static SDValue splitStoreSplat ( SelectionDAG &  DAG, StoreSDNode &  St, SDValue  SplatVal, unsigned  NumVecElts  )

static

Definition at line 10094 of file AArch64ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::MemSDNode::getAlignment(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::i64, llvm::StoreSDNode::isTruncatingStore(), and llvm::MinAlign().

Referenced by replaceSplatVectorStore(), and replaceZeroVectorStore().

STATISTIC() [1/3]

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"
	)

STATISTIC() [2/3]

STATISTIC	(	NumShiftInserts	,
		"Number of vector shift inserts"
	)

STATISTIC() [3/3]

STATISTIC	(	NumOptimizedImms	,
		"Number of times immediates were optimized"
	)

tryAdvSIMDModImm16()

static SDValue tryAdvSIMDModImm16 ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits, const SDValue *  LHS = nullptr  )

static

Definition at line 6827 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType5(), llvm::AArch64_AM::encodeAdvSIMDModImmType6(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::getZExtValue(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType5(), llvm::AArch64_AM::isAdvSIMDModImmType6(), llvm::AArch64ISD::NVCAST, llvm::MVT::v4i16, llvm::MVT::v8i16, and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector(), tryAdvSIMDModImmFP(), and tryLowerToSLI().

tryAdvSIMDModImm32()

static SDValue tryAdvSIMDModImm32 ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits, const SDValue *  LHS = nullptr  )

static

Definition at line 6779 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType1(), llvm::AArch64_AM::encodeAdvSIMDModImmType2(), llvm::AArch64_AM::encodeAdvSIMDModImmType3(), llvm::AArch64_AM::encodeAdvSIMDModImmType4(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::getZExtValue(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType1(), llvm::AArch64_AM::isAdvSIMDModImmType2(), llvm::AArch64_AM::isAdvSIMDModImmType3(), llvm::AArch64_AM::isAdvSIMDModImmType4(), llvm::AArch64ISD::NVCAST, llvm::MVT::v2i32, llvm::MVT::v4i32, and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector(), tryAdvSIMDModImmFP(), and tryLowerToSLI().

tryAdvSIMDModImm321s()

static SDValue tryAdvSIMDModImm321s ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits  )

static

Definition at line 6867 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType7(), llvm::AArch64_AM::encodeAdvSIMDModImmType8(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::getZExtValue(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType7(), llvm::AArch64_AM::isAdvSIMDModImmType8(), llvm::AArch64ISD::NVCAST, llvm::MVT::v2i32, llvm::MVT::v4i32, and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector().

tryAdvSIMDModImm64()

static SDValue tryAdvSIMDModImm64 ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits  )

static

Definition at line 6758 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType10(), llvm::MVT::f64, llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::getZExtValue(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType10(), llvm::AArch64ISD::NVCAST, llvm::MVT::v2i64, and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector().

tryAdvSIMDModImm8()

static SDValue tryAdvSIMDModImm8 ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits  )

static

Definition at line 6898 of file AArch64ISelLowering.cpp.

References llvm::AArch64_AM::encodeAdvSIMDModImmType9(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::getZExtValue(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType9(), llvm::AArch64ISD::NVCAST, llvm::MVT::v16i8, llvm::MVT::v8i8, and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector().

tryAdvSIMDModImmFP()

static SDValue tryAdvSIMDModImmFP ( unsigned  NewOp, SDValue  Op, SelectionDAG &  DAG, const APInt &  Bits  )

static

Definition at line 6919 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::BICi, llvm::dyn_cast(), llvm::AArch64_AM::encodeAdvSIMDModImmType11(), llvm::AArch64_AM::encodeAdvSIMDModImmType12(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::getZExtValue(), llvm::MVT::i32, llvm::AArch64_AM::isAdvSIMDModImmType11(), llvm::AArch64_AM::isAdvSIMDModImmType12(), llvm::AArch64ISD::NVCAST, resolveBuildVector(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), llvm::MVT::v2f32, llvm::MVT::v2f64, llvm::MVT::v4f32, and llvm::APInt::zextOrTrunc().

Referenced by ConstantBuildVector().

tryCombineCRC32()

static SDValue tryCombineCRC32 ( unsigned  Mask, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 9917 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, and llvm::BitmaskEnumDetail::Mask().

Referenced by performIntrinsicCombine().

tryCombineFixedPointConvert()

static SDValue tryCombineFixedPointConvert ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 9532 of file AArch64ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm_unreachable, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, and llvm::MVT::v4i32.

Referenced by performIntrinsicCombine().

tryCombineLongOpWithDup()

static SDValue tryCombineLongOpWithDup ( unsigned  IID, SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, SelectionDAG &  DAG  )

static

Definition at line 9825 of file AArch64ISelLowering.cpp.

References assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractSubvector(), and tryExtendDUPToExtractHigh().

Referenced by performExtendCombine(), and performIntrinsicCombine().

tryCombineShiftImm()

static SDValue tryCombineShiftImm ( unsigned  IID, SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 9854 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_sqshl, llvm::Intrinsic::aarch64_neon_sqshlu, llvm::Intrinsic::aarch64_neon_srshl, llvm::Intrinsic::aarch64_neon_uqshl, llvm::Intrinsic::aarch64_neon_urshl, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::APInt::getSExtValue(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm_unreachable, llvm::AArch64ISD::SQSHL_I, llvm::AArch64ISD::SQSHLU_I, llvm::AArch64ISD::SRSHR_I, llvm::AArch64ISD::UQSHL_I, and llvm::AArch64ISD::URSHR_I.

Referenced by performIntrinsicCombine().

tryCombineToBSL()

static SDValue tryCombineToBSL ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 9289 of file AArch64ISelLowering.cpp.

References llvm::ISD::AND, llvm::tgtok::Bits, llvm::AArch64ISD::BSL, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), and llvm::EVT::isVector().

Referenced by performORCombine().

tryCombineToEXTR()

static SDValue tryCombineToEXTR ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

EXTR instruction extracts a contiguous chunk of bits from two existing registers viewed as a high/low pair.

This function looks for the pattern: (or (shl VAL1, #N), (srl VAL2, #RegWidth-N)) and replaces it with an EXTR. Can't quite be done in TableGen because the two immediates aren't independent.

Definition at line 9249 of file AArch64ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::AArch64ISD::EXTR, findEXTRHalf(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::OR, and std::swap().

Referenced by performORCombine().

tryExtendDUPToExtractHigh()

static SDValue tryExtendDUPToExtractHigh ( SDValue  N, SelectionDAG &  DAG  )

static

Definition at line 9594 of file AArch64ISelLowering.cpp.

References llvm::AArch64ISD::DUP, llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i64, llvm::MVT::is64BitVector(), llvm::AArch64ISD::MOVI, llvm::AArch64ISD::MOVIedit, llvm::AArch64ISD::MOVImsl, llvm::AArch64ISD::MOVIshift, llvm::AArch64ISD::MVNImsl, llvm::AArch64ISD::MVNIshift, and llvm::SDNode::ops().

Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().

tryFormConcatFromShuffle()

static SDValue tryFormConcatFromShuffle ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 6375 of file AArch64ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::i64, isConcatMask(), and llvm::BitmaskEnumDetail::Mask().

Referenced by getDUPLANEOp().

tryLowerToSLI()

static SDValue tryLowerToSLI ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 7027 of file AArch64ISelLowering.cpp.

References llvm::Intrinsic::aarch64_neon_vsli, llvm::Intrinsic::aarch64_neon_vsri, llvm::ISD::AND, llvm::dbgs(), llvm::SDNode::dump(), llvm::dyn_cast(), EnableAArch64SlrGeneration, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i32, llvm::ISD::INTRINSIC_WO_CHAIN, isAllConstantBuildVector(), llvm::EVT::isVector(), LLVM_DEBUG, llvm::AArch64ISD::ORRi, resolveBuildVector(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), llvm::AArch64ISD::VLSHR, llvm::AArch64ISD::VSHL, X, and Y.

UseTlsOffset()

static Value* UseTlsOffset ( IRBuilder<> &  IRB, unsigned  Offset  )

static

Definition at line 11711 of file AArch64ISelLowering.cpp.

References llvm::IRBuilder< T, Inserter >::CreateCall(), llvm::IRBuilder< T, Inserter >::CreateConstGEP1_32(), llvm::IRBuilder< T, Inserter >::CreatePointerCast(), llvm::IRBuilderBase::getContext(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::GetInsertBlock(), llvm::Type::getInt8PtrTy(), llvm::BasicBlock::getParent(), llvm::GlobalValue::getParent(), and llvm::Intrinsic::thread_pointer.

Referenced by llvm::AArch64TargetLowering::getIRStackGuard(), and llvm::AArch64TargetLowering::getSafeStackPointerLocation().

WidenVector()

static SDValue WidenVector ( SDValue  V64Reg, SelectionDAG &  DAG  )

static

WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.

Definition at line 5887 of file AArch64ISelLowering.cpp.

References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i32, and llvm::ISD::INSERT_SUBVECTOR.

Referenced by ConstantBuildVector(), GeneratePerfectShuffle(), getDUPLANEOp(), NarrowVector(), and performConcatVectorsCombine().

Variable Documentation

EnableAArch64ELFLocalDynamicTLSGeneration

cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false))

Referenced by mayTailCallThisCC().

EnableAArch64SlrGeneration

cl::opt<bool> EnableAArch64SlrGeneration("aarch64-shift-insert-generation", cl::Hidden, cl::desc("Allow AArch64 SLI/SRI formation"), cl::init(false))

static

Referenced by tryLowerToSLI().

EnableOptimizeLogicalImm

cl::opt<bool> EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true))

static

Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().

MVT_CC

const MVT MVT_CC = MVT::i32

static

Value type used for condition codes.

Definition at line 117 of file AArch64ISelLowering.cpp.