#include "X86ISelLowering.h"#include "Utils/X86ShuffleDecode.h"#include "X86CallingConv.h"#include "X86FrameLowering.h"#include "X86InstrBuilder.h"#include "X86IntrinsicsInfo.h"#include "X86MachineFunctionInfo.h"#include "X86TargetMachine.h"#include "X86TargetObjectFile.h"#include "llvm/ADT/SmallBitVector.h"#include "llvm/ADT/SmallSet.h"#include "llvm/ADT/Statistic.h"#include "llvm/ADT/StringExtras.h"#include "llvm/ADT/StringSwitch.h"#include "llvm/Analysis/EHPersonalities.h"#include "llvm/CodeGen/IntrinsicLowering.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineFunction.h"#include "llvm/CodeGen/MachineInstrBuilder.h"#include "llvm/CodeGen/MachineJumpTableInfo.h"#include "llvm/CodeGen/MachineModuleInfo.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/TargetLowering.h"#include "llvm/CodeGen/WinEHFuncInfo.h"#include "llvm/IR/CallSite.h"#include "llvm/IR/CallingConv.h"#include "llvm/IR/Constants.h"#include "llvm/IR/DerivedTypes.h"#include "llvm/IR/DiagnosticInfo.h"#include "llvm/IR/Function.h"#include "llvm/IR/GlobalAlias.h"#include "llvm/IR/GlobalVariable.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/Intrinsics.h"#include "llvm/MC/MCAsmInfo.h"#include "llvm/MC/MCContext.h"#include "llvm/MC/MCExpr.h"#include "llvm/MC/MCSymbol.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Debug.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/KnownBits.h"#include "llvm/Support/MathExtras.h"#include "llvm/Target/TargetOptions.h"#include <algorithm>#include <bitset>#include <cctype>#include <numeric>#include "X86GenCallingConv.inc"
Go to the source code of this file.
Macros | |
| #define | DEBUG_TYPE "x86-isel" |
Enumerations | |
| enum | StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn } |
| CallIsStructReturn - Determines whether a call uses struct return semantics. More... | |
| enum | ShrinkMode { MULS8, MULU8, MULS16, MULU16 } |
| Different mul shrinking modes. More... | |
Functions | |
| STATISTIC (NumTailCalls, "Number of tail calls") | |
| static void | errorUnsupported (SelectionDAG &DAG, const SDLoc &dl, const char *Msg) |
| Call this when the user attempts to do something unsupported, like returning a double without SSE2 enabled on x86_64. More... | |
| static void | getMaxByValAlign (Type *Ty, unsigned &MaxAlign) |
| Helper for getByValTypeAlignment to determine the desired ByVal argument alignment. More... | |
| static bool | hasStackGuardSlotTLS (const Triple &TargetTriple) |
| static Constant * | SegmentOffset (IRBuilder<> &IRB, unsigned Offset, unsigned AddressSpace) |
| static SDValue | lowerMasksToReg (const SDValue &ValArg, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG) |
| Lowers masks values (v*i1) to the local register values. More... | |
| static void | Passv64i1ArgInRegs (const SDLoc &Dl, SelectionDAG &DAG, SDValue Chain, SDValue &Arg, SmallVector< std::pair< unsigned, SDValue >, 8 > &RegsToPass, CCValAssign &VA, CCValAssign &NextVA, const X86Subtarget &Subtarget) |
| Breaks v64i1 value into two registers and adds the new node to the DAG. More... | |
| static SDValue | getv64i1Argument (CCValAssign &VA, CCValAssign &NextVA, SDValue &Root, SelectionDAG &DAG, const SDLoc &Dl, const X86Subtarget &Subtarget, SDValue *InFlag=nullptr) |
| Reads two 32 bit registers and creates a 64 bit mask value. More... | |
| static SDValue | lowerRegToMasks (const SDValue &ValArg, const EVT &ValVT, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG) |
| The function will lower a register of various sizes (8/16/32/64) to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1) More... | |
| static StructReturnType | callIsStructReturn (ArrayRef< ISD::OutputArg > Outs, bool IsMCU) |
| static StructReturnType | argsAreStructReturn (ArrayRef< ISD::InputArg > Ins, bool IsMCU) |
| Determines whether a function uses struct return semantics. More... | |
| static SDValue | CreateCopyOfByValArgument (SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl) |
| Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute. More... | |
| static bool | canGuaranteeTCO (CallingConv::ID CC) |
| Return true if the calling convention is one that we can guarantee TCO for. More... | |
| static bool | mayTailCallThisCC (CallingConv::ID CC) |
| Return true if we might ever do TCO for calls with this calling convention. More... | |
| static bool | shouldGuaranteeTCO (CallingConv::ID CC, bool GuaranteedTailCallOpt) |
| Return true if the function is being made into a tailcall target by changing its ABI. More... | |
| static ArrayRef< MCPhysReg > | get64BitArgumentGPRs (CallingConv::ID CallConv, const X86Subtarget &Subtarget) |
| static ArrayRef< MCPhysReg > | get64BitArgumentXMMs (MachineFunction &MF, CallingConv::ID CallConv, const X86Subtarget &Subtarget) |
| static bool | isSortedByValueNo (ArrayRef< CCValAssign > ArgLocs) |
| static SDValue | EmitTailCallStoreRetAddr (SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT, unsigned SlotSize, int FPDiff, const SDLoc &dl) |
| Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0). More... | |
| static SDValue | getMOVL (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2) |
| Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width. More... | |
| static bool | MatchingStackOffset (SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const X86InstrInfo *TII, const CCValAssign &VA) |
| Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack. More... | |
| static bool | MayFoldLoad (SDValue Op) |
| static bool | MayFoldIntoStore (SDValue Op) |
| static bool | MayFoldIntoZeroExtend (SDValue Op) |
| static bool | isTargetShuffle (unsigned Opcode) |
| static bool | isTargetShuffleVariableMask (unsigned Opcode) |
| static bool | isX86CCUnsigned (unsigned X86CC) |
| Return true if the condition is an unsigned comparison operation. More... | |
| static X86::CondCode | TranslateIntegerX86CC (ISD::CondCode SetCCOpcode) |
| static X86::CondCode | TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) |
| Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make. More... | |
| static bool | hasFPCMov (unsigned X86CC) |
| Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. More... | |
| static bool | isUndefOrEqual (int Val, int CmpVal) |
| Val is the undef sentinel value or equal to the specified value. More... | |
| static bool | isUndefOrZero (int Val) |
| Val is either the undef or zero sentinel value. More... | |
| static bool | isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size) |
| Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value. More... | |
| static bool | isInRange (int Val, int Low, int Hi) |
| Return true if Val falls within the specified range (L, H]. More... | |
| static bool | isAnyInRange (ArrayRef< int > Mask, int Low, int Hi) |
| Return true if the value of any element in Mask falls within the specified range (L, H]. More... | |
| static bool | isUndefOrInRange (int Val, int Low, int Hi) |
| Return true if Val is undef or if its value falls within the specified range (L, H]. More... | |
| static bool | isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi) |
| Return true if every element in Mask is undef or if its value falls within the specified range (L, H]. More... | |
| static bool | isUndefOrZeroOrInRange (int Val, int Low, int Hi) |
| Return true if Val is undef, zero or if its value falls within the specified range (L, H]. More... | |
| static bool | isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi) |
| Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H]. More... | |
| static bool | isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1) |
| Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef. More... | |
| static bool | isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low) |
| Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero. More... | |
| static bool | isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size) |
| Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero. More... | |
| static bool | canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask) |
| Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled. More... | |
| static bool | canWidenShuffleElements (ArrayRef< int > Mask, const APInt &Zeroable, SmallVectorImpl< int > &WidenedMask) |
| static bool | canWidenShuffleElements (ArrayRef< int > Mask) |
| static SDValue | getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false) |
| static SDValue | getConstVector (ArrayRef< APInt > Bits, APInt &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl) |
| static SDValue | getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
| Returns a vector of specified type with all zero elements. More... | |
| static SDValue | extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth) |
| static SDValue | extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
| Generate a DAG to grab 128-bits from a vector > 128 bits. More... | |
| static SDValue | extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
| Generate a DAG to grab 256-bits from a 512-bit vector. More... | |
| static SDValue | insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth) |
| static SDValue | insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
| Generate a DAG to put 128-bits into a vector > 128 bits. More... | |
| static SDValue | widenSubVector (MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
| Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef. More... | |
| template<typename F > | |
| SDValue | SplitOpsAndApply (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, F Builder, bool CheckBWI=true) |
| static bool | isMaskedZeroUpperBitsvXi1 (unsigned int Opcode) |
| static SDValue | insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Insert i1-subvector to i1-vector. More... | |
| static SDValue | concatSubVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, const SDLoc &dl, unsigned VectorWidth) |
| static SDValue | getOnesVector (EVT VT, SelectionDAG &DAG, const SDLoc &dl) |
| Returns a vector of specified type with all bits set. More... | |
| static SDValue | getExtendInVec (bool Signed, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG) |
| static SDValue | getUnpackl (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2) |
| Returns a vector_shuffle node for an unpackl operation. More... | |
| static SDValue | getUnpackh (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2) |
| Returns a vector_shuffle node for an unpackh operation. More... | |
| static SDValue | getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Return a vector_shuffle of the specified vector of zero or undef vector. More... | |
| static SDValue | peekThroughEXTRACT_SUBVECTORs (SDValue V) |
| static const Constant * | getTargetConstantFromNode (SDValue Op) |
| static bool | getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, APInt &UndefElts, SmallVectorImpl< APInt > &EltBits, bool AllowWholeUndefs=true, bool AllowPartialUndefs=true) |
| static bool | isConstantSplat (SDValue Op, APInt &SplatVal) |
| static bool | getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask, APInt &UndefElts) |
| static void | createPackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Unary) |
| Create a shuffle mask that matches the PACKSS/PACKUS truncation. More... | |
| static void | getPackDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS) |
| static bool | getTargetShuffleMask (SDNode *N, MVT VT, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary) |
| Calculates the shuffle mask corresponding to the target-specific opcode. More... | |
| static bool | setTargetShuffleZeroElements (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops) |
| Check a target shuffle mask's inputs to see if we can set any values to SM_SentinelZero - this is for elements that are known to be zero (not just zeroable) from their inputs. More... | |
| static bool | resolveTargetShuffleInputs (SDValue Op, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG) |
| Calls setTargetShuffleZeroElements to resolve a target shuffle mask's inputs and set the SM_SentinelUndef and SM_SentinelZero values. More... | |
| static bool | getFauxShuffleMask (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, const SelectionDAG &DAG) |
| static void | resolveTargetShuffleInputsAndMask (SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask) |
| Removes unused shuffle source inputs and adjusts the shuffle mask accordingly. More... | |
| static SDValue | getShuffleScalarElt (SDNode *N, unsigned Index, SelectionDAG &DAG, unsigned Depth) |
| Returns the scalar element that will make up the ith element of the result of the vector shuffle. More... | |
| static SDValue | LowerBuildVectorAsInsert (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerBuildVectorv16i8 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Custom lower build_vector of v16i8. More... | |
| static SDValue | LowerBuildVectorv8i16 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Custom lower build_vector of v8i16. More... | |
| static SDValue | LowerBuildVectorv4x32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Custom lower build_vector of v4i32 or v4f32. More... | |
| static SDValue | getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl) |
| Return a vector logical shift node. More... | |
| static SDValue | LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG) |
| static SDValue | EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool isAfterLegalize) |
| Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'. More... | |
| static Constant * | getConstantVector (MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C) |
| static bool | isUseOfShuffle (SDNode *N) |
| static SDValue | isSplatZeroExtended (const BuildVectorSDNode *Op, unsigned &NumElt, MVT &EltType) |
| static SDValue | lowerBuildVectorAsBroadcast (BuildVectorSDNode *BVOp, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a. More... | |
| static int | getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx) |
| For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. More... | |
| static SDValue | buildFromShuffleMostly (SDValue Op, SelectionDAG &DAG) |
| static SDValue | ConvertI1VectorToInteger (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerBUILD_VECTORvXi1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static bool | isHorizontalBinOpPart (const BuildVectorSDNode *N, unsigned Opcode, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1) |
| This is a helper function of LowerToHorizontalOp(). More... | |
| static SDValue | ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI) |
| Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. More... | |
| static bool | isAddSubOrSubAdd (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, unsigned &NumExtracts, bool &IsSubAdd) |
Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation. More... | |
| static bool | isFMAddSubOrFMSubAdd (const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2, unsigned ExpectedUses) |
Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1). More... | |
| static SDValue | lowerToAddSubOrFMAddSub (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node. More... | |
| static bool | isHopBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned &HOpcode, SDValue &V0, SDValue &V1) |
| static SDValue | getHopForBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned HOpcode, SDValue V0, SDValue V1) |
| static SDValue | LowerToHorizontalOp (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower BUILD_VECTOR to a horizontal add/sub operation if possible. More... | |
| static SDValue | lowerBuildVectorToBitOp (BuildVectorSDNode *Op, SelectionDAG &DAG) |
| If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors. More... | |
| static SDValue | materializeVectorConstant (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Create a vector constant without a load. More... | |
| static SDValue | createVariablePermute (MVT VT, SDValue SrcVec, SDValue IndicesVec, SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices. More... | |
| static SDValue | LowerBUILD_VECTORAsVariablePermute (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static bool | isExpandWithZeros (const SDValue &Op) |
| static SDValue | isTypePromotionOfi1ZeroUpBits (SDValue Op) |
| static SDValue | LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | isNoopShuffleMask (ArrayRef< int > Mask) |
| Tiny helper function to identify a no-op mask. More... | |
| static bool | is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask) |
| Test whether there are elements crossing 128-bit lanes in this shuffle mask. More... | |
| static bool | isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a shuffle mask is equivalent within each sub-lane. More... | |
| static bool | is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a shuffle mask is equivalent within each 128-bit lane. More... | |
| static bool | is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask) |
| static bool | is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a shuffle mask is equivalent within each 256-bit lane. More... | |
| static bool | isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a target shuffle mask is equivalent within each sub-lane. More... | |
| static bool | isShuffleEquivalent (SDValue V1, SDValue V2, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask) |
| Checks whether a shuffle mask is equivalent to an explicit list of arguments. More... | |
| static bool | isTargetShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask) |
| Checks whether a target shuffle mask is equivalent to an explicit pattern. More... | |
| static SmallVector< int, 64 > | createTargetShuffleMask (ArrayRef< int > Mask, const APInt &Zeroable) |
| static bool | createShuffleMaskFromVSELECT (SmallVectorImpl< int > &Mask, SDValue Cond) |
| static bool | isUnpackWdShuffleMask (ArrayRef< int > Mask, MVT VT) |
| static unsigned | getV4X86ShuffleImm (ArrayRef< int > Mask) |
| Get a 4-lane 8-bit shuffle immediate for a mask. More... | |
| static SDValue | getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG) |
| static APInt | computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2) |
| Compute whether each element of a shuffle is zeroable. More... | |
| static bool | isNonZeroElementsInOrder (const APInt &Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft) |
| static SDValue | lowerVectorShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a shuffle with a single PSHUFB of V1 or V2. More... | |
| static SDValue | getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics. More... | |
| static SDValue | lowerVectorShuffleToEXPAND (const SDLoc &DL, MVT VT, const APInt &Zeroable, ArrayRef< int > Mask, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static bool | matchVectorShuffleWithUNPCK (MVT VT, SDValue &V1, SDValue &V2, unsigned &UnpackOpcode, bool IsUnary, ArrayRef< int > TargetMask, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | lowerVectorShuffleWithUNPCK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| static bool | matchVectorShuffleAsVPMOV (ArrayRef< int > Mask, bool SwappedOps, int Delta) |
| static SDValue | lowerVectorShuffleWithVPMOV (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static bool | matchVectorShuffleWithPACK (MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, unsigned &PackOpcode, ArrayRef< int > TargetMask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | lowerVectorShuffleWithPACK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | lowerVectorShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG) |
| Try to emit a bitmask instruction for a shuffle. More... | |
| static SDValue | lowerVectorShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to emit a blend instruction for a shuffle using bit math. More... | |
| static SDValue | getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics. More... | |
| static bool | matchVectorShuffleAsBlend (SDValue V1, SDValue V2, MutableArrayRef< int > TargetMask, bool &ForceV1Zero, bool &ForceV2Zero, uint64_t &BlendMask) |
| static uint64_t | scaleVectorShuffleBlendMask (uint64_t BlendMask, int Size, int Scale) |
| static SDValue | lowerVectorShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to emit a blend instruction for a shuffle. More... | |
| static SDValue | lowerVectorShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool ImmBlends=false) |
| Try to lower as a blend of elements from two inputs followed by a single-input permutation. More... | |
| static SDValue | lowerVectorShuffleAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to lower as an unpack of elements from two inputs followed by a single-input permutation. More... | |
| static SDValue | lowerVectorShuffleAsByteRotateAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place. More... | |
| static SDValue | lowerVectorShuffleAsDecomposedShuffleBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Generic routine to decompose a shuffle and blend into independent blends and permutes. More... | |
| static int | matchVectorShuffleAsRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask) |
| Try to lower a vector shuffle as a rotation. More... | |
| static int | matchVectorShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask) |
| Try to lower a vector shuffle as a byte rotation. More... | |
| static SDValue | lowerVectorShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleAsRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a dword/qword rotation. More... | |
| static int | matchVectorShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable, const X86Subtarget &Subtarget) |
| Try to lower a vector shuffle as a bit shift (shifts in zeros). More... | |
| static SDValue | lowerVectorShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | matchVectorShuffleAsEXTRQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx, const APInt &Zeroable) |
| static bool | matchVectorShuffleAsINSERTQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx) |
| static SDValue | lowerVectorShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG) |
| Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. More... | |
| static SDValue | lowerVectorShuffleAsSpecificZeroOrAnyExtend (const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower a vector shuffle as a zero or any extension. More... | |
| static SDValue | lowerVectorShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a zero extension on any microarch. More... | |
| static SDValue | getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG) |
| Try to get a scalar value for a specific element of a vector. More... | |
| static bool | isShuffleFoldableLoad (SDValue V) |
| Helper to test for a load that can be folded with x86 shuffles. More... | |
| static SDValue | lowerVectorShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower insertion of a single element into a zero vector. More... | |
| static SDValue | lowerVectorShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements. More... | |
| static SDValue | lowerVectorShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower broadcast of a single element. More... | |
| static bool | matchVectorShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const APInt &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction. More... | |
| static SDValue | lowerV2F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 2-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV2I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 2-lane 64-bit integer shuffles. More... | |
| static bool | isSingleSHUFPSMask (ArrayRef< int > Mask) |
| Test whether this can be lowered with a single SHUFPS instruction. More... | |
| static SDValue | lowerVectorShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| Lower a vector shuffle using the SHUFPS instruction. More... | |
| static SDValue | lowerV4F32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower 4-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV4I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower 4-lane i32 vector shuffles. More... | |
| static SDValue | lowerV8I16GeneralSingleInputVectorShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. More... | |
| static SDValue | lowerVectorShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) |
| Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used. More... | |
| static SDValue | lowerV8I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Generic lowering of 8-lane i16 shuffles. More... | |
| static int | canLowerByDroppingEvenElements (ArrayRef< int > Mask, bool IsSingleInput) |
| Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped. More... | |
| static SDValue | lowerVectorShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| static SDValue | lowerV16I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Generic lowering of v16i8 shuffles. More... | |
| static SDValue | lower128BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Dispatching routine to lower various 128-bit x86 vector shuffles. More... | |
| static SDValue | splitAndLowerVectorShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Generic routine to split vector shuffle into half-sized shuffles. More... | |
| static SDValue | lowerVectorShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Either split a vector in halves or decompose the shuffles and the blend. More... | |
| static SDValue | lowerVectorShuffleAsLanePermuteAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation. More... | |
| static SDValue | lowerVectorShuffleAsLanePermuteAndBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes. More... | |
| static SDValue | lowerV2X128VectorShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering 2-lane 128-bit shuffles. More... | |
| static SDValue | lowerVectorShuffleByMerging128BitLanes (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane. More... | |
| static SDValue | lowerVectorShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF. More... | |
| static bool | isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask) |
| Test whether the specified input (0 or 1) is in-place blended by the given mask. More... | |
| static SDValue | lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes. More... | |
| static bool | matchVectorShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, unsigned &ShuffleImm, ArrayRef< int > Mask) |
| static SDValue | lowerVectorShuffleWithSHUFPD (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| static SDValue | lowerV4F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 4-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV4I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 4-lane 64-bit integer shuffles. More... | |
| static SDValue | lowerV8F32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV8I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 32-bit integer shuffles. More... | |
| static SDValue | lowerV16I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 16-bit integer shuffles. More... | |
| static SDValue | lowerV32I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 32-lane 8-bit integer shuffles. More... | |
| static SDValue | lower256BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| High-level routine to lower various 256-bit x86 vector shuffles. More... | |
| static SDValue | lowerV4X128VectorShuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a 128-bit shuffles. More... | |
| static SDValue | lowerV8F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV16F32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV8I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 64-bit integer shuffles. More... | |
| static SDValue | lowerV16I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 32-bit integer shuffles. More... | |
| static SDValue | lowerV32I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 32-lane 16-bit integer shuffles. More... | |
| static SDValue | lowerV64I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 64-lane 8-bit integer shuffles. More... | |
| static SDValue | lower512BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| High-level routine to lower various 512-bit x86 vector shuffles. More... | |
| static int | match1BitShuffleAsKSHIFT (unsigned &Opcode, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable) |
| static SDValue | lower1BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask) |
| Helper function that returns true if the shuffle mask should be commuted to improve canonicalization. More... | |
| static SDValue | lowerVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Top-level lowering for x86 vector shuffles. More... | |
| static SDValue | lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a VSELECT instruction to a vector shuffle. More... | |
| static SDValue | LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG) |
| static SDValue | ExtractBitFromMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Extract one bit from mask vector, like v16i1 or v8i1. More... | |
| static SDValue | InsertBitToMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Insert one bit to mask vector, like v16i1 or v8i1. More... | |
| static SDValue | LowerSCALAR_TO_VECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false) |
| static SDValue | LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
| static SDValue | LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
| static SDValue | LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool is64Bit) |
| static SDValue | LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC) |
| static SDValue | LowerShiftParts (SDValue Op, SelectionDAG &DAG) |
| Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount. More... | |
| static SDValue | LowerFunnelShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerI64IntToFP_AVX512DQ (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerUINT_TO_FP_i64 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| 64-bit unsigned integer to double expansion. More... | |
| static SDValue | LowerUINT_TO_FP_i32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| 32-bit unsigned integer to float expansion. More... | |
| static SDValue | lowerUINT_TO_FP_v2i32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL) |
| static SDValue | lowerUINT_TO_FP_vXi32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | lowerUINT_TO_FP_vec (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | SplitAndExtendv16i1 (unsigned ExtOpc, MVT VT, SDValue In, const SDLoc &dl, SelectionDAG &DAG) |
| static SDValue | LowerZERO_EXTEND_Mask (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | truncateVectorWithPACK (unsigned Opcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Helper to recursively truncate vector elements in half with PACKSS/PACKUS. More... | |
| static SDValue | LowerTruncateVecI1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerFP_EXTEND (SDValue Op, SelectionDAG &DAG) |
| static bool | shouldUseHorizontalOp (bool IsSingleSource, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Horizontal vector math instructions may be slower than normal math with shuffles. More... | |
| static SDValue | lowerAddSubToHorizontalOp (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation. More... | |
| static SDValue | lowerFaddFsub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation. More... | |
| static SDValue | LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG) |
| The only differences between FABS and FNEG are the mask and the logic op. More... | |
| static SDValue | LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerFGETSIGN (SDValue Op, SelectionDAG &DAG) |
| static SDValue | getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG) |
| Helper for creating a X86ISD::SETCC node. More... | |
| static SDValue | LowerVectorAllZeroTest (SDValue Op, ISD::CondCode CC, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &X86CC) |
| static bool | hasNonFlagsUse (SDValue Op) |
return true if Op has a use that doesn't just read flags. More... | |
| static SDValue | EmitTest (SDValue Op, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Emit nodes that will be selected as "test Op0,Op0", or something equivalent. More... | |
| static SDValue | LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, SDValue &X86CC) |
| Result of 'and' is compared against zero. More... | |
| static unsigned | translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1) |
| Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs. More... | |
| static SDValue | Lower256IntVSETCC (SDValue Op, SelectionDAG &DAG) |
| Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back. More... | |
| static SDValue | LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG) |
| static SDValue | decrementVectorConstant (SDValue V, SelectionDAG &DAG) |
| Given a simple buildvector constant, return a new vector constant with each element decremented. More... | |
| static SDValue | LowerVSETCCWithSUBUS (SDValue Op0, SDValue Op1, MVT VT, ISD::CondCode Cond, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| As another special case, use PSUBUS[BW] when it's profitable. More... | |
| static SDValue | LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | EmitKORTEST (SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue &X86CC) |
| static std::pair< SDValue, SDValue > | getX86XALUOOp (X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerXALUO (SDValue Op, SelectionDAG &DAG) |
| static bool | isX86LogicalCmp (SDValue Op) |
| Return true if opcode is a X86 logical comparison. More... | |
| static bool | isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG) |
| static SDValue | LowerSIGN_EXTEND_Mask (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerStore (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | isAndOrOfSetCCs (SDValue Op, unsigned &Opc) |
| Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR. More... | |
| static bool | isXor1OfSetCC (SDValue Op) |
| Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the SETCC node has a single use. More... | |
| static SDValue | LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static unsigned | getTargetVShiftUniformOpcode (unsigned Opc, bool IsVariable) |
| static SDValue | getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG) |
| Handle vector element shifts where the shift amount is a constant. More... | |
| static SDValue | getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle vector element shifts where the shift amount may or may not be a constant. More... | |
| static SDValue | getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Creates an SDNode for a predicated scalar operation. More... | |
| static int | getSEHRegistrationNodeSize (const Function *Fn) |
| static SDValue | recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP) |
| When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP. More... | |
| static SDValue | getAVX2GatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
| static SDValue | getGatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
| static SDValue | getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
| static SDValue | getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
| static void | getExtendedControlRegister (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results) |
| Handles the lowering of builtin intrinsic that return the value of the extended control register. More... | |
| static void | getReadPerformanceCounter (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results) |
| Handles the lowering of builtin intrinsics that read performance monitor counters (x86_rdpmc). More... | |
| static void | getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results) |
| Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp). More... | |
| static SDValue | LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG) |
| static SDValue | MarkEHGuard (SDValue Op, SelectionDAG &DAG) |
| static SDValue | EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG) |
| Emit Truncating Store with signed or unsigned saturation. More... | |
| static SDValue | EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG) |
| Emit Masked Truncating Store with signed or unsigned saturation. More... | |
| static SDValue | LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerVectorIntUnary (SDValue Op, SelectionDAG &DAG) |
| static SDValue | Lower256IntUnary (SDValue Op, SelectionDAG &DAG) |
| static SDValue | Lower512IntUnary (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTLZ_AVX512CDI (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Lower a vector CTLZ using native supported vector CTLZ instruction. More... | |
| static SDValue | LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCTTZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | split256IntArith (SDValue Op, SelectionDAG &DAG) |
| Break a 256-bit integer operation into two new 128-bit ones and then concatenate the result back. More... | |
| static SDValue | split512IntArith (SDValue Op, SelectionDAG &DAG) |
| Break a 512-bit integer operation into two new 256-bit ones and then concatenate the result back. More... | |
| static SDValue | lowerAddSub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerADDSAT_SUBSAT (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerABS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMINMAX (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | SupportedVectorShiftWithImm (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
| static bool | SupportedVectorShiftWithBaseAmnt (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
| static bool | SupportedVectorVarShift (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
| static SDValue | LowerScalarImmediateShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | IsSplatVector (SDValue V, int &SplatIdx, SelectionDAG &DAG) |
| static SDValue | GetSplatValue (SDValue V, const SDLoc &dl, SelectionDAG &DAG) |
| static SDValue | LowerScalarVariableShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | convertShiftLeftToScale (SDValue Amt, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | getPMOVMSKB (const SDLoc &DL, SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Compute the horizontal sum of bytes in V for the elements of VT. More... | |
| static SDValue | LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Lower atomic_load_ops into LOCK-prefixed operations. More... | |
| static SDValue | LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerADDSUBCARRY (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerFSINCOS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false) |
| Widen a vector input to a vector of NVT. More... | |
| static SDValue | LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static MachineBasicBlock * | emitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII) |
| Utility function to emit xbegin specifying the start of an RTM region. More... | |
| static MachineBasicBlock * | emitWRPKRU (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget) |
| static MachineBasicBlock * | emitRDPKRU (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget) |
| static MachineBasicBlock * | emitMonitor (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget, unsigned Opc) |
| static MachineBasicBlock * | emitClzero (MachineInstr *MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget) |
| static bool | checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI) |
| static bool | isCMOVPseudo (MachineInstr &MI) |
| static MachineInstrBuilder | createPHIsForCMOVsInSinkBB (MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB) |
| static unsigned | getOpcodeForRetpoline (unsigned RPOpc) |
| static const char * | getRetpolineSymbol (const X86Subtarget &Subtarget, unsigned Reg) |
| static bool | matchUnaryVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT) |
| static bool | matchUnaryPermuteVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) |
| static bool | matchBinaryVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary) |
| static bool | matchBinaryPermuteVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) |
| static SDValue | combineX86ShuffleChain (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Combine an arbitrary chain of shuffles into a single instruction if possible. More... | |
| static SDValue | combineX86ShufflesConstants (ArrayRef< SDValue > Ops, ArrayRef< int > Mask, SDValue Root, bool HasVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, ArrayRef< const SDNode *> SrcNodes, unsigned Depth, bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Fully generic combining of x86 shuffle instructions. More... | |
| static SmallVector< int, 4 > | getPSHUFShuffleMask (SDValue N) |
| Get the PSHUF-style mask from PSHUF node. More... | |
| static SDValue | combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG) |
| Search for a combinable shuffle across a chain ending in pshufd. More... | |
| static SDValue | combineTargetShuffle (SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Try to combine x86 target specific shuffles. More... | |
| static bool | isAddSubOrSubAddMask (ArrayRef< int > Mask, bool &Op0Even) |
| Checks if the shuffle mask takes subsequent elements alternately from two vectors. More... | |
| static bool | isAddSubOrSubAdd (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, bool &IsSubAdd) |
Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation. More... | |
| static SDValue | combineShuffleToFMAddSub (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Combine shuffle of two fma nodes into FMAddSub or FMSubAdd. More... | |
| static SDValue | combineShuffleToAddSubOrFMAddSub (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to combine a shuffle into a target-specific add-sub or mul-add-sub node. More... | |
| static SDValue | combineShuffleOfConcatUndef (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | foldShuffleOfHorizOp (SDNode *N) |
| Eliminate a redundant shuffle of a horizontal math op. More... | |
| static SDValue | combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | XFormVExtractWithShuffleIntoLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| Check if a vector extract from a target-specific shuffle of a load can be folded into a single element load. More... | |
| static SDValue | combineBitcastvxi1 (SelectionDAG &DAG, SDValue BitCast, const X86Subtarget &Subtarget) |
| static SDValue | combinevXi1ConstantToInteger (SDValue Op, SelectionDAG &DAG) |
| static SDValue | combineCastedMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | createMMXBuildVector (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineBitcast (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static bool | detectZextAbsDiff (const SDValue &Select, SDValue &Op0, SDValue &Op1) |
| static SDValue | createPSADBW (SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL, const X86Subtarget &Subtarget) |
| static SDValue | combineHorizontalMinMaxResult (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineHorizontalPredicateResult (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineExtractWithShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence. More... | |
| static SDValue | combineVSelectWithAllOnesOrZeros (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation. More... | |
| static SDValue | combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineVSelectToBLENDV (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element. More... | |
| static SDValue | combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on SELECT and VSELECT nodes. More... | |
| static SDValue | combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction. More... | |
| static SDValue | checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC) |
| static bool | checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd) |
| Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS. More... | |
| static SDValue | combineCarryThroughADD (SDValue EFLAGS) |
| static SDValue | combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values. More... | |
| static SDValue | combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. More... | |
| static bool | canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) |
| static SDValue | reduceVMULWidth (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code. More... | |
| static SDValue | combineMulSpecial (uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, EVT VT, const SDLoc &DL) |
| static SDValue | combineMulToPMADDWD (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineMulToPMULDQ (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Optimize a single multiply with constant into two operations in order to implement it with two cheaper instructions, e.g. More... | |
| static SDValue | combineShiftLeft (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineShiftRightArithmetic (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineShiftRightLogical (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineShift (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineVectorPack (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineVectorShiftVar (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineVectorShiftImm (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineVectorInsert (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends. More... | |
| static SDValue | combineANDXORWithAllOnesIntoANDNP (SDNode *N, SelectionDAG &DAG) |
| Try to fold: (and (xor X, -1), Y) -> (andnp X, Y). More... | |
| static SDValue | PromoteMaskArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | convertIntLogicToFPLogic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| If both input operands of a logic op are being cast from floating point types, try to convert this into a floating point logic node to avoid unnecessary moves from SSE to integer registers. More... | |
| static SDValue | combineAndMaskToShift (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| If this is a zero/all-bits result that is bitwise-anded with a low bits mask. More... | |
| static SDValue | getIndexFromUnindexedLoad (LoadSDNode *Ld) |
| static bool | hasBZHI (const X86Subtarget &Subtarget, MVT VT) |
| static SDValue | combineAndLoadToBZHI (SDNode *Node, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineParity (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static bool | matchLogicBlend (SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) |
| static SDValue | combineLogicBlendIntoPBLENDV (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | lowerX86CmpEqZeroToCtlzSrl (SDValue Op, EVT ExtTy, SelectionDAG &DAG) |
| static SDValue | combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | foldXorTruncShiftIntoCmp (SDNode *N, SelectionDAG &DAG) |
| Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1) More... | |
| static SDValue | foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1. More... | |
| static bool | isSATValidOnAVX512Subtarget (EVT SrcVT, EVT DstVT, const X86Subtarget &Subtarget) |
Check if truncation with saturation form type SrcVT to DstVT is valid for the given Subtarget. More... | |
| static SDValue | detectUSatPattern (SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL) |
| Detect patterns of truncation with unsigned saturation: More... | |
| static SDValue | detectSSatPattern (SDValue In, EVT VT, bool MatchPackUS=false) |
| Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type). More... | |
| static SDValue | detectAVX512SSatPattern (SDValue In, EVT VT, const X86Subtarget &Subtarget, const TargetLowering &TLI) |
| Detect a pattern of truncation with signed saturation. More... | |
| static SDValue | detectAVX512USatPattern (SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget, const TargetLowering &TLI) |
| Detect a pattern of truncation with saturation: (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type). More... | |
| static SDValue | combineTruncateWithSat (SDValue In, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | detectAVGPattern (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL) |
| This function detects the AVG pattern between vectors of unsigned i8/i16, which is c = (a + b + 1) / 2, and replace this operation with the efficient X86ISD::AVG instruction. More... | |
| static SDValue | combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static int | getOneTrueElt (SDValue V) |
| If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element. More... | |
| static bool | getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, unsigned &Alignment) |
| Given a masked memory load/store operation, return true if it has one mask bit set. More... | |
| static SDValue | reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert. More... | |
| static SDValue | combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG) |
| If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store. More... | |
| static SDValue | combineMaskedStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineStore (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static bool | isHorizontalBinOp (SDValue &LHS, SDValue &RHS, bool IsCommutative) |
| Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. More... | |
| static SDValue | combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on floating-point adds/subs. More... | |
| static SDValue | combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL) |
| Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen. More... | |
| static SDValue | combineVectorTruncationWithPACKUS (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Truncate using ISD::AND mask and X86ISD::PACKUS. More... | |
| static SDValue | combineVectorTruncationWithPACKSS (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS. More... | |
| static SDValue | combineVectorTruncation (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations. More... | |
| static SDValue | combineVectorSignBitsTruncation (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| This function transforms vector truncation of 'extended sign-bits' or 'extended zero-bits' values. More... | |
| static SDValue | combinePMULH (SDValue Src, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | detectPMADDUBSW (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL) |
| static SDValue | combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | isFNEG (SelectionDAG &DAG, SDNode *N) |
Returns the negated value if the node N flips sign of FP value. More... | |
| static SDValue | combineFneg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on floating point negations. More... | |
| static SDValue | lowerX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | foldXor1SetCC (SDNode *N, SelectionDAG &DAG) |
| Fold a xor(setcc cond, val), 1 –> setcc (inverted(cond), val) More... | |
| static SDValue | combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineBEXTR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static bool | isNullFPScalarOrVectorConst (SDValue V) |
| static SDValue | getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value. More... | |
| static SDValue | combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on X86ISD::FAND nodes. More... | |
| static SDValue | combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on X86ISD::FANDN nodes. More... | |
| static SDValue | combineFOr (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes. More... | |
| static SDValue | combineFMinFMax (SDNode *N, SelectionDAG &DAG) |
| Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes. More... | |
| static SDValue | combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineX86INT_TO_FP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineAndnp (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on X86ISD::ANDNP nodes. More... | |
| static SDValue | combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineSextInRegCmov (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| sext(add_nsw(x, C)) –> add(sext(x), C_sext) zext(add_nuw(x, C)) –> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode. More... | |
| static SDValue | combineToExtendCMOV (SDNode *Extend, SelectionDAG &DAG) |
| static SDValue | combineToExtendBoolVectorInReg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineToExtendVectorInReg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Convert a SEXT or ZEXT of a vector to a SIGN_EXTEND_VECTOR_INREG or ZERO_EXTEND_VECTOR_INREG, this requires the splitting (or concatenating with UNDEFs) of the input to vectors of the same size as the target type which then extends the lowest elements. More... | |
| static SDValue | combineExtSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static unsigned | negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc) |
| static SDValue | combineFMA (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineFMADDSUB (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineVectorSizedSetCCEquality (SDNode *SetCC, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks. More... | |
| static SDValue | combineSetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineMOVMSK (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineGatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineX86SetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineBrCond (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Optimize branch condition evaluation. More... | |
| static SDValue | combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineSIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static bool | needCarryOrOverflowFlag (SDValue Flags) |
| static bool | onlyZeroFlagUsed (SDValue Flags) |
| static SDValue | combineCMP (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineSBB (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineADC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineAddOrSubToADCOrSBB (SDNode *N, SelectionDAG &DAG) |
| If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB. More... | |
| static SDValue | combineLoopMAddPattern (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineLoopSADPattern (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineIncDecVector (SDNode *N, SelectionDAG &DAG) |
| Convert vector increment or decrement to sub/add with an all-ones constant: add X, <1, 1...> –> sub X, <-1, -1...> sub X, <1, 1...> –> add X, <-1, -1...> The all-ones vector constant can be materialized using a pcmpeq instruction that is commonly recognized as an idiom (has no register dependency), so that's better/smaller than loading a splat 1 constant. More... | |
| static SDValue | matchPMADDWD (SelectionDAG &DAG, SDValue Op0, SDValue Op1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget) |
| static SDValue | combineAddToSUBUS (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | matchPMADDWD_2 (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget) |
| static SDValue | combineAdd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineSubToSubus (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineInsertSubvector (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineExtractSubvector (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineScalarToVector (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combinePMULDQ (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static bool | matchAsm (StringRef S, ArrayRef< const char *> Pieces) |
| static bool | clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces) |
| static bool | isGRClass (const TargetRegisterClass &RC) |
Check if RC is a general purpose register class. More... | |
| static bool | isFRClass (const TargetRegisterClass &RC) |
Check if RC is a vector register class. More... | |
Variables | |
| static cl::opt< bool > | ExperimentalVectorWideningLegalization ("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening " "rather than promotion."), cl::Hidden) |
| static cl::opt< int > | ExperimentalPrefLoopAlignment ("x86-experimental-pref-loop-alignment", cl::init(4), cl::desc("Sets the preferable loop alignment for experiments " "(the last x86-experimental-pref-loop-alignment bits" " of the loop header PC will be 0)."), cl::Hidden) |
| static cl::opt< bool > | MulConstantOptimization ("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden) |
Macro Definition Documentation
◆ DEBUG_TYPE
| #define DEBUG_TYPE "x86-isel" |
Definition at line 65 of file X86ISelLowering.cpp.
Enumeration Type Documentation
◆ ShrinkMode
| enum ShrinkMode |
Different mul shrinking modes.
| Enumerator | |
|---|---|
| MULS8 | |
| MULU8 | |
| MULS16 | |
| MULU16 | |
Definition at line 34983 of file X86ISelLowering.cpp.
◆ StructReturnType
| enum StructReturnType |
CallIsStructReturn - Determines whether a call uses struct return semantics.
| Enumerator | |
|---|---|
| NotStructReturn | |
| RegStructReturn | |
| StackStructReturn | |
Definition at line 2872 of file X86ISelLowering.cpp.
Function Documentation
◆ argsAreStructReturn()
|
static |
Determines whether a function uses struct return semantics.
Definition at line 2892 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::empty(), llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.
Referenced by isSortedByValueNo().
◆ buildFromShuffleMostly()
|
static |
Definition at line 7684 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getUnderlyingExtractedFromVec(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::INSERT_VECTOR_ELT, llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::BitmaskEnumDetail::Mask(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorBase::size(), and llvm::ISD::UNDEF.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ callIsStructReturn()
|
static |
Definition at line 2878 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::empty(), llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.
Referenced by getMOVL().
◆ canGuaranteeTCO()
|
static |
Return true if the calling convention is one that we can guarantee TCO for.
Definition at line 2919 of file X86ISelLowering.cpp.
References llvm::CallingConv::Fast, llvm::CallingConv::GHC, llvm::CallingConv::HHVM, llvm::CallingConv::HiPE, and llvm::CallingConv::X86_RegCall.
Referenced by getMOVL(), isSortedByValueNo(), MatchingStackOffset(), mayTailCallThisCC(), and shouldGuaranteeTCO().
◆ canLowerByDroppingEvenElements()
Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped.
This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:
N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24
Any of these lanes can of course be undef.
This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.
- Returns
- N above, or the number of times even elements must be dropped if there is such a number. Otherwise returns zero.
Definition at line 13292 of file X86ISelLowering.cpp.
References llvm::array_lengthof(), assert(), llvm::isPowerOf2_32(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I8VectorShuffle().
◆ canonicalizeShuffleMaskWithCommute()
Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.
Definition at line 15952 of file X86ISelLowering.cpp.
References assert(), Size, and llvm::ArrayRef< T >::size().
Referenced by combineX86ShufflesRecursively(), and lowerVectorShuffle().
◆ canReduceVMulWidth()
|
static |
Definition at line 34985 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), MULS16, MULS8, MULU16, MULU8, and llvm::SelectionDAG::SignBitIsZero().
Referenced by combineLoopMAddPattern(), matchPMADDWD(), and reduceVMULWidth().
◆ canWidenShuffleElements() [1/3]
|
static |
Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.
Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.
NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.
Definition at line 5146 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), Size, llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by canWidenShuffleElements(), combineExtractWithShuffle(), combineX86ShufflesRecursively(), LowerShift(), lowerV2X128VectorShuffle(), lowerV4X128VectorShuffle(), lowerVectorShuffle(), and lowerVectorShuffleAsBlendAndPermute().
◆ canWidenShuffleElements() [2/3]
|
static |
Definition at line 5196 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::begin(), canWidenShuffleElements(), llvm::ArrayRef< T >::end(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
◆ canWidenShuffleElements() [3/3]
Definition at line 5209 of file X86ISelLowering.cpp.
References canWidenShuffleElements().
◆ checkAndUpdateEFLAGSKill()
|
static |
Definition at line 28143 of file X86ISelLowering.cpp.
References llvm::MachineInstr::definesRegister(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::isLiveIn(), llvm::MachineInstr::readsRegister(), llvm::MachineBasicBlock::succ_begin(), and llvm::MachineBasicBlock::succ_end().
Referenced by createPHIsForCMOVsInSinkBB().
◆ checkBoolTestAndOrSetCCCombine()
|
static |
Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)
Definition at line 34688 of file X86ISelLowering.cpp.
References llvm::X86ISD::AND, llvm::ISD::AND, llvm::X86ISD::CMP, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::isNullConstant(), LLVM_FALLTHROUGH, llvm::X86ISD::OR, llvm::ISD::OR, and llvm::X86ISD::SETCC.
Referenced by combineCMov().
◆ checkBoolTestSetCCCombine()
|
static |
Definition at line 34570 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), C, llvm::X86ISD::CMOV, llvm::X86ISD::CMP, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::dyn_cast(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::SDValue::getResNo(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasAnyUseOfValue(), llvm::isOneConstant(), LLVM_FALLTHROUGH, llvm::X86ISD::RDRAND, llvm::X86ISD::RDSEED, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSetCCEFLAGS().
◆ clobbersFlagRegisters()
|
static |
Definition at line 41814 of file X86ISelLowering.cpp.
References llvm::SmallVectorTemplateCommon< T >::begin(), llvm::count(), llvm::SmallVectorTemplateCommon< T >::end(), and llvm::SmallVectorBase::size().
Referenced by llvm::X86TargetLowering::ExpandInlineAsm().
◆ combineADC()
|
static |
Definition at line 40367 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::ISD::AND, combineCarryThroughADD(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::X86::COND_B, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::MVT::i8, llvm::X86::isZeroNode(), llvm::X86ISD::SETCC_CARRY, and llvm::SDValue::use_empty().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineAdd()
|
static |
Definition at line 40986 of file X86ISelLowering.cpp.
References combineAddOrSubToADCOrSBB(), combineAddToSUBUS(), combineIncDecVector(), combineLoopMAddPattern(), combineLoopSADPattern(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasSSSE3(), llvm::SDNodeFlags::hasVectorReduction(), isHorizontalBinOp(), llvm::MipsISD::MAdd, matchPMADDWD(), matchPMADDWD_2(), shouldUseHorizontalOp(), SplitOpsAndApply(), llvm::MVT::v16i16, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineAddOrSubToADCOrSBB()
|
static |
If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
This replaces TEST+SET+{ADD/SUB} with CMP+{ADC, SBB}.
Definition at line 40403 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::CMP, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::MVT::i8, llvm::EVT::isInteger(), llvm::X86::isZeroNode(), llvm::X86ISD::SBB, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SUB, llvm::X86ISD::SUB, std::swap(), X, Y, llvm::Z, and llvm::ISD::ZERO_EXTEND.
Referenced by combineAdd(), and combineSub().
◆ combineAddToSUBUS()
|
static |
Definition at line 40833 of file X86ISelLowering.cpp.
References llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i8, llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::ISD::matchBinaryPredicate(), llvm::ISD::UMAX, and llvm::ISD::USUBSAT.
Referenced by combineAdd().
◆ combineAnd()
|
static |
Definition at line 36354 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::AND, combineAndLoadToBZHI(), combineAndMaskToShift(), combineANDXORWithAllOnesIntoANDNP(), combineCompareEqual(), combineParity(), combineX86ShufflesRecursively(), convertIntLogicToFPLogic(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), getTargetConstantBitsFromNode(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is64Bit(), llvm::APInt::isAllOnesValue(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNullValue(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineAndLoadToBZHI()
|
static |
Definition at line 36209 of file X86ISelLowering.cpp.
Referenced by combineAnd().
◆ combineAndMaskToShift()
|
static |
If this is a zero/all-bits result that is bitwise-anded with a low bits mask.
(Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to eliminate loading the vector constant mask value.
Definition at line 36135 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::ComputeNumSignBits(), llvm::APInt::countTrailingOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i8, llvm::isBitwiseNot(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isMask(), llvm::EVT::isSimple(), llvm::peekThroughBitcasts(), llvm::ISD::SRL, SupportedVectorShiftWithImm(), and llvm::X86ISD::VSRLI.
Referenced by combineAnd().
◆ combineAndnp()
|
static |
Do target-specific dag combines on X86ISD::ANDNP nodes.
Definition at line 39008 of file X86ISelLowering.cpp.
References llvm::ISD::AND, combineX86ShufflesRecursively(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDNode::getSimpleValueType(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isVector(), and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineANDXORWithAllOnesIntoANDNP()
|
static |
Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
Definition at line 36002 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::ISD::isBuildVectorAllOnes(), llvm::peekThroughBitcasts(), X, llvm::ISD::XOR, and Y.
Referenced by combineAnd().
◆ combineBasicSADPattern()
|
static |
Definition at line 33564 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, createPSADBW(), detectZextAbsDiff(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::isSimple(), llvm::Log2_32(), llvm::BitmaskEnumDetail::Mask(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::SIGN_EXTEND, llvm::X86Subtarget::useBWIRegs(), llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.
Referenced by combineExtractVectorElt().
◆ combineBEXTR()
|
static |
Definition at line 38782 of file X86ISelLowering.cpp.
References llvm::X86ISD::BEXTR, llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineBitcast()
|
static |
Definition at line 33141 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, C, combineBitcastvxi1(), combineCastedMaskArithmetic(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::countLeadingZeros(), createMMXBuildVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_TO_SINT, llvm::X86ISD::FXOR, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, llvm::APFloatBase::IEEEdouble(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), llvm::EVT::isScalarInteger(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, llvm::ISD::OR, llvm::ISD::TRUNCATE, llvm::MVT::v2f32, llvm::MVT::v2i1, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i1, llvm::MVT::v4i16, llvm::MVT::v4i32, llvm::MVT::v8i1, llvm::MVT::v8i8, llvm::MVT::x86mmx, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineBitcastvxi1()
|
static |
Definition at line 32888 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), getPMOVMSKB(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::EVT::isScalarInteger(), llvm::EVT::isSimple(), llvm::MipsISD::Lo, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::MVT::SimpleTy, llvm::SelectionDAG::SplitVector(), llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v16i8, llvm::MVT::v2i1, llvm::MVT::v2i64, llvm::MVT::v32i1, llvm::MVT::v32i8, llvm::MVT::v4i1, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v64i1, llvm::MVT::v64i8, llvm::MVT::v8i1, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::ISD::ZERO_EXTEND.
Referenced by combineBitcast().
◆ combineBrCond()
|
static |
Optimize branch condition evaluation.
Definition at line 40041 of file X86ISelLowering.cpp.
References llvm::X86ISD::BRCOND, combineSetCCEFLAGS(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getVTList(), and llvm::MVT::i8.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineBT()
|
static |
Definition at line 39040 of file X86ISelLowering.cpp.
References llvm::X86ISD::BT, llvm::SelectionDAG::GetDemandedBits(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::i32, and llvm::Log2_32().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineCarryThroughADD()
Definition at line 34729 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::X86::COND_B, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::isAllOnesConstant(), llvm::isOneConstant(), llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineADC(), combineSBB(), and combineSetCCEFLAGS().
◆ combineCastedMaskArithmetic()
|
static |
Definition at line 33012 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, combinevXi1ConstantToInteger(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by combineBitcast().
◆ combineCMov()
|
static |
Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].
Definition at line 34767 of file X86ISelLowering.cpp.
References llvm::MCID::Add, llvm::ISD::ADD, checkBoolTestAndOrSetCCCombine(), llvm::ARCISD::CMOV, llvm::X86ISD::CMOV, llvm::X86ISD::CMP, combineSetCCEFLAGS(), llvm::X86::COND_E, llvm::X86::COND_NE, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_UNDEF, llvm::MVT::f80, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), hasFPCMov(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::ISD::MUL, llvm::ISD::SHL, llvm::ISD::SUB, llvm::X86ISD::SUB, std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineCMP()
|
static |
Definition at line 40265 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::X86ISD::AND, llvm::ISD::AND, llvm::X86ISD::CMP, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::isNullConstant(), llvm::APInt::isSignedIntN(), needCarryOrOverflowFlag(), onlyZeroFlagUsed(), llvm::X86ISD::OR, llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::SRL, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, llvm::X86ISD::XOR, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineCompareEqual()
|
static |
Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends.
Likewise for OR -> CMPNEQSS.
Definition at line 35898 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BR_CC, llvm::ISD::BRCOND, llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_P, llvm::ISD::CopyToReg, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FSETCC, llvm::X86ISD::FSETCCM, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::X86Subtarget::is64Bit(), isAndOrOfSetCCs(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SELECT, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v16i1, llvm::MVT::v1i1, llvm::MVT::v2f64, llvm::MVT::v4f32, and llvm::ISD::ZERO_EXTEND.
Referenced by combineAnd(), and combineOr().
◆ combineExtractSubvector()
|
static |
Definition at line 41338 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, Concat, llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTSI2P, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::MVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDNode::ops(), llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFB, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::SINT_TO_FP, split256IntArith(), llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::X86ISD::VFPEXT, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineExtractVectorElt()
|
static |
Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.
For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.
Definition at line 33749 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, combineBasicSADPattern(), combineExtractWithShuffle(), combineHorizontalMinMaxResult(), combineHorizontalPredicateResult(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::isNullConstant(), llvm::X86ISD::MMX_MOVD2W, llvm::MVT::v1i64, llvm::MVT::v2i32, llvm::MVT::x86mmx, and XFormVExtractWithShuffleIntoLoad().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineExtractWithShuffle()
|
static |
Definition at line 33647 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i32, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::MipsISD::Lo, llvm::BitmaskEnumDetail::Mask(), llvm::peekThroughBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, resolveTargetShuffleInputs(), llvm::SmallVectorBase::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::X86ISD::VBROADCAST.
Referenced by combineExtractVectorElt().
◆ combineExtSetcc()
|
static |
Definition at line 39467 of file X86ISelLowering.cpp.
References llvm::EVT::changeVectorElementTypeToInteger(), llvm::MVT::f32, llvm::MVT::f64, llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), llvm::ISD::SETCC, Size, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
◆ combineFaddFsub()
|
static |
Do target-specific dag combines on floating-point adds/subs.
Definition at line 38090 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), isHorizontalBinOp(), shouldUseHorizontalOp(), llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, and llvm::MVT::v8f32.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineFAnd()
|
static |
Do target-specific dag combines on X86ISD::FAND nodes.
Definition at line 38864 of file X86ISelLowering.cpp.
References combineFAndFNotToFAndn(), getNullFPConstForNullVal(), llvm::SDNode::getOperand(), and lowerX86FPLogicOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineFAndFNotToFAndn()
|
static |
Definition at line 38832 of file X86ISelLowering.cpp.
References C, llvm::dyn_cast(), llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FANDN, llvm::X86ISD::FXOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), and llvm::MVT::v4f32.
Referenced by combineFAnd().
◆ combineFAndn()
|
static |
Do target-specific dag combines on X86ISD::FANDN nodes.
Definition at line 38881 of file X86ISelLowering.cpp.
References getNullFPConstForNullVal(), llvm::SDNode::getOperand(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineFMA()
|
static |
Definition at line 39583 of file X86ISelLowering.cpp.
References B, C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAnyFMA(), isFNEG(), llvm::isNullConstant(), llvm::TargetLoweringBase::isTypeLegal(), and negateFMAOpcode().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineFMADDSUB()
|
static |
Definition at line 39637 of file X86ISelLowering.cpp.
References llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMADDSUB_RND, llvm::X86ISD::FMSUBADD, llvm::X86ISD::FMSUBADD_RND, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), isFNEG(), and llvm_unreachable.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineFMinFMax()
|
static |
Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
Definition at line 38914 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::FMAX, llvm::X86ISD::FMAXC, llvm::X86ISD::FMIN, llvm::X86ISD::FMINC, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTarget(), llvm::SDNode::getValueType(), llvm_unreachable, llvm::TargetMachine::Options, and llvm::TargetOptions::UnsafeFPMath.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineFMinNumFMaxNum()
|
static |
Definition at line 38934 of file X86ISelLowering.cpp.
References llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FMAX, llvm::ISD::FMAXNUM, llvm::X86ISD::FMIN, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDNode::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), llvm::SDNodeFlags::hasNoNaNs(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::isVector(), llvm::TargetOptions::NoNaNsFPMath, llvm::Function::optForMinSize(), llvm::TargetMachine::Options, llvm::ISD::SETUO, llvm::X86Subtarget::useSoftFloat(), llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, and llvm::MVT::v8f32.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineFneg()
|
static |
Do target-specific dag combines on floating point negations.
Definition at line 38657 of file X86ISelLowering.cpp.
References llvm::MVT::f32, llvm::MVT::f64, llvm::ISD::FMA, llvm::X86ISD::FMADD_RND, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUB_RND, llvm::ISD::FMUL, llvm::X86ISD::FNMADD, llvm::X86ISD::FNMADD_RND, llvm::X86ISD::FNMSUB, llvm::X86ISD::FNMSUB_RND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNodeFlags::hasNoSignedZeros(), llvm::SDValue::hasOneUse(), isFNEG(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::SDNode::ops().
Referenced by combineFOr(), combineXor(), and llvm::X86TargetLowering::PerformDAGCombine().
◆ combineFOr()
|
static |
Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
Definition at line 38895 of file X86ISelLowering.cpp.
References assert(), combineFneg(), llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineGatherScatter()
|
static |
Definition at line 39953 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::SelectionDAG::getContext(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::i32, llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::SignBitIsZero(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::SelectionDAG::UpdateNodeOperands(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineHorizontalMinMaxResult()
|
static |
Definition at line 33414 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i16, llvm::MVT::i8, llvm::MipsISD::Lo, llvm::BitmaskEnumDetail::Mask(), llvm::SelectionDAG::matchBinOpReduction(), llvm::X86ISD::PHMINPOS, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::Upper, llvm::MVT::v16i8, llvm::MVT::v8i16, and llvm::ISD::XOR.
Referenced by combineExtractVectorElt().
◆ combineHorizontalPredicateResult()
|
static |
Definition at line 33491 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::ComputeNumSignBits(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MVT::getFloatingPointVT(), llvm::EVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::APInt::getNullValue(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSelectCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::Match, llvm::SelectionDAG::matchBinOpReduction(), llvm::max(), llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::ISD::SETEQ, and llvm::ISD::SETNE.
Referenced by combineExtractVectorElt().
◆ combineIncDecVector()
|
static |
Convert vector increment or decrement to sub/add with an all-ones constant: add X, <1, 1...> –> sub X, <-1, -1...> sub X, <1, 1...> –> add X, <-1, -1...> The all-ones vector constant can be materialized using a pcmpeq instruction that is commonly recognized as an idiom (has no register dependency), so that's better/smaller than loading a splat 1 constant.
Definition at line 40704 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), isConstantSplat(), llvm::APInt::isOneValue(), and llvm::ISD::SUB.
Referenced by combineAdd(), and combineSub().
◆ combineInsertSubvector()
|
static |
Definition at line 41176 of file X86ISelLowering.cpp.
References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::ISD::BITCAST, EltsFromConsecutiveLoads(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::CallingConv::Fast, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::X86Subtarget::getTargetLowering(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), and llvm::X86ISD::SUBV_BROADCAST.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineLoad()
|
static |
Definition at line 37246 of file X86ISelLowering.cpp.
References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, llvm::MipsISD::Ext, llvm::CallingConv::Fast, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::LoadSDNode::getExtensionType(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::X86Subtarget::hasInt256(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::MemSDNode::isNonTemporal(), llvm::MinAlign(), N, llvm::ISD::NON_EXTLOAD, llvm::MVT::Other, and llvm::ISD::TokenFactor.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineLogicBlendIntoPBLENDV()
|
static |
Definition at line 36494 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isOperationLegal(), llvm::BitmaskEnumDetail::Mask(), matchLogicBlend(), N, llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::ISD::SUB, std::swap(), llvm::MVT::v16i8, llvm::MVT::v32i8, X, llvm::ISD::XOR, and Y.
Referenced by combineOr().
◆ combineLoopMAddPattern()
|
static |
Definition at line 40559 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, canReduceVMulWidth(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, llvm::EVT::isVector(), Mode, llvm::ISD::MUL, SplitOpsAndApply(), std::swap(), llvm::ISD::TRUNCATE, and llvm::X86ISD::VPMADDWD.
Referenced by combineAdd().
◆ combineLoopSADPattern()
|
static |
Definition at line 40619 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::BITCAST, createPSADBW(), detectZextAbsDiff(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i32, llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::isSimple(), llvm::EVT::isVector(), std::swap(), llvm::ISD::TRUNCATE, llvm::X86Subtarget::useBWIRegs(), and llvm::ISD::VSELECT.
Referenced by combineAdd().
◆ combineMaskedLoad()
|
static |
Definition at line 37430 of file X86ISelLowering.cpp.
References assert(), combineMaskedLoadConstantMask(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, llvm::MaskedLoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), getExtendInVec(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::MaskedLoadSDNode::getPassThru(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::i1, llvm::MaskedLoadSDNode::isExpandingLoad(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), N, llvm::ISD::NON_EXTLOAD, reduceMaskedLoadToScalarLoad(), and llvm::ISD::SEXTLOAD.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineMaskedLoadConstantMask()
|
static |
Definition at line 37386 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::MaskedLoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::SelectionDAG::getLoad(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::MaskedLoadSDNode::getPassThru(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), and llvm::SDValue::isUndef().
Referenced by combineMaskedLoad().
◆ combineMaskedStore()
|
static |
Definition at line 37549 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::MaskedStoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::APInt::getSignMask(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::i1, llvm::MaskedStoreSDNode::isCompressingStore(), llvm::isPowerOf2_32(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::BitmaskEnumDetail::Mask(), N, reduceMaskedStoreToScalarStore(), and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineMOVMSK()
|
static |
Definition at line 39885 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::tgtok::In, llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isConstantSplatVector(), llvm::MVT::isFloatingPoint(), llvm::APInt::isPowerOf2(), llvm::SDValue::isUndef(), llvm::APInt::logBase2(), llvm::X86ISD::MOVMSK, llvm::APInt::setBit(), llvm::ISD::SETCC, llvm::ISD::SETNE, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), and X.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineMul()
|
static |
Optimize a single multiply with constant into two operations in order to implement it with two cheaper instructions, e.g.
LEA + SHL, LEA + LEA.
Definition at line 35349 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), C, combineMulSpecial(), combineMulToPMADDWD(), combineMulToPMULDQ(), llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::ConstantSDNode::getSExtValue(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, INT64_MIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::isPowerOf2_64(), llvm::EVT::isVector(), llvm::Log2_64(), llvm::X86ISD::MUL_IMM, MulConstantOptimization, N, llvm::Function::optForMinSize(), reduceVMULWidth(), llvm::ISD::SHL, llvm::X86Subtarget::slowLEA(), llvm::ISD::SUB, std::swap(), UINT64_MAX, and llvm::SDNode::use_begin().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineMulSpecial()
|
static |
Definition at line 35168 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::countTrailingZeros(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::i8, llvm::isPowerOf2_64(), llvm::Log2_64(), llvm::X86ISD::MUL_IMM, llvm::MipsISD::Mult, llvm::ISD::SHL, and llvm::ISD::SUB.
Referenced by combineMul().
◆ combineMulToPMADDWD()
|
static |
Definition at line 35256 of file X86ISelLowering.cpp.
References ExperimentalVectorWideningLegalization, llvm::SelectionDAG::getBitcast(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::X86Subtarget::isPMADDWDSlow(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), SplitOpsAndApply(), llvm::MVT::v2i32, llvm::X86ISD::VPMADDWD, and llvm::ISD::ZERO_EXTEND.
Referenced by combineMul().
◆ combineMulToPMULDQ()
|
static |
Definition at line 35305 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::ComputeNumSignBits(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i64, llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, and SplitOpsAndApply().
Referenced by combineMul().
◆ combineOr()
|
static |
Definition at line 36692 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::tgtok::Bits, combineCompareEqual(), combineLogicBlendIntoPBLENDV(), combineX86ShufflesRecursively(), convertIntLogicToFPLogic(), llvm::dyn_cast(), llvm::X86ISD::FOR, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::X86Subtarget::isSHLDSlow(), llvm::EVT::isVector(), llvm::Function::optForSize(), llvm::ISD::SHL, llvm::X86ISD::SHLD, llvm::X86ISD::SHRD, llvm::ISD::SRL, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineOrCmpEqZeroToCtlzSrl()
|
static |
Definition at line 36612 of file X86ISelLowering.cpp.
References llvm::EVT::bitsGE(), llvm::MVT::bitsGE(), llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::X86Subtarget::getTargetLowering(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::MVT::i32, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::X86TargetLowering::isCtlzFast(), llvm::isNullConstant(), lowerX86CmpEqZeroToCtlzSrl(), N, OR, llvm::ISD::OR, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::MipsISD::Ret, llvm::X86ISD::SETCC, llvm::SmallVectorBase::size(), std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineZext().
◆ combineParity()
|
static |
Definition at line 36289 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::X86::COND_NP, llvm::ISD::CTPOP, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), getSETCC(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isOneConstant(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MipsISD::Lo, llvm::ISD::SRL, llvm::ISD::TRUNCATE, X, llvm::X86ISD::XOR, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by combineAnd().
◆ combinePMULDQ()
|
static |
Definition at line 41469 of file X86ISelLowering.cpp.
References llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::GetDemandedBits(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combinePMULH()
|
static |
Definition at line 38356 of file X86ISelLowering.cpp.
References ExperimentalVectorWideningLegalization, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.
Referenced by combineTruncate().
◆ combineRedundantDWordShuffle()
|
static |
Search for a combinable shuffle across a chain ending in pshufd.
We walk up the chain and look for a combinable shuffle, skipping over shuffles that we could hoist this shuffle's transformation past without altering anything.
Definition at line 31618 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::SmallVectorBase::empty(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::MVT::i16, llvm::MVT::i8, llvm::SDNode::isOnlyUserOf(), LLVM_FALLTHROUGH, llvm_unreachable, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::RISCVFenceField::W.
Referenced by combineTargetShuffle().
◆ combineSBB()
|
static |
Definition at line 40354 of file X86ISelLowering.cpp.
References combineCarryThroughADD(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, and llvm::X86ISD::SBB.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineScalarToVector()
|
static |
Definition at line 41442 of file X86ISelLowering.cpp.
References llvm::ISD::AND, C, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::EVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, and llvm::MVT::v1i1.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineSelect()
|
static |
Do target-specific dag combines on SELECT and VSELECT nodes.
Definition at line 34067 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, combineSelectOfTwoConstants(), combineVSelectToBLENDV(), combineVSelectWithAllOnesOrZeros(), createShuffleMaskFromVSELECT(), ExperimentalVectorWideningLegalization, llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::f80, llvm::X86ISD::FMAX, llvm::X86ISD::FMIN, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i64, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZeroFloat(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LLVM_FALLTHROUGH, llvm::BitmaskEnumDetail::Mask(), llvm::ISD::matchBinaryPredicate(), llvm::TargetMachine::Options, Other, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::simplifySelect(), llvm::ISD::SUB, std::swap(), llvm::ISD::UADDSAT, llvm::TargetOptions::UnsafeFPMath, llvm::ISD::USUBSAT, llvm::MVT::v2f32, llvm::ISD::VSELECT, llvm::MVT::x86mmx, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineSelectOfTwoConstants()
|
static |
Definition at line 33921 of file X86ISelLowering.cpp.
References llvm::APInt::abs(), llvm::ISD::ADD, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::APInt::isOneValue(), llvm::APInt::isPowerOf2(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::MUL, llvm::APInt::slt(), llvm::APInt::ssub_ov(), std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineSelect().
◆ combineSetCC()
|
static |
Definition at line 39802 of file X86ISelLowering.cpp.
References llvm::MCID::Add, llvm::ISD::ADD, assert(), llvm::ISD::BUILD_VECTOR, combineVectorSizedSetCCEquality(), ExperimentalVectorWideningLegalization, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::ISD::isBuildVectorAllZeros(), llvm::isNullConstant(), llvm::ISD::isSignedIntSetCC(), llvm::EVT::isVector(), LowerVSETCC(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, llvm::MVT::v4f32, and llvm::MVT::v4i32.
Referenced by llvm::PPCTargetLowering::functionArgumentNeedsConsecutiveRegisters(), and llvm::X86TargetLowering::PerformDAGCombine().
◆ combineSetCCAtomicArith()
|
static |
Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction.
Note that this is only legal for some op/cc combinations.
Definition at line 34463 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_SUB, llvm::X86ISD::CMP, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NS, llvm::X86::COND_S, llvm::dyn_cast(), llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDValue::hasOneUse(), llvm::APInt::isNullValue(), lowerAtomicArithWithLOCK(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::X86ISD::SUB.
Referenced by combineSetCCEFLAGS().
◆ combineSetCCEFLAGS()
|
static |
Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.
Definition at line 34754 of file X86ISelLowering.cpp.
References checkBoolTestSetCCCombine(), combineCarryThroughADD(), combineSetCCAtomicArith(), and llvm::X86::COND_B.
Referenced by combineBrCond(), combineCMov(), and combineX86SetCC().
◆ combineSext()
|
static |
Definition at line 39508 of file X86ISelLowering.cpp.
References combineExtSetcc(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), combineToExtendVectorInReg(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isVector(), promoteExtBeforeAdd(), PromoteMaskArithmetic(), llvm::ISD::SUB, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineSextInRegCmov()
|
static |
Definition at line 39055 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::X86ISD::CMOV, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i16, and llvm::ISD::SIGN_EXTEND_INREG.
Referenced by combineSignExtendInReg().
◆ combineShift()
|
static |
Definition at line 35652 of file X86ISelLowering.cpp.
References combineShiftLeft(), combineShiftRightArithmetic(), combineShiftRightLogical(), llvm::SDNode::getOpcode(), llvm::ISD::SHL, llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineShiftLeft()
|
static |
Definition at line 35491 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::Constant, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::isIntN(), llvm::EVT::isVector(), llvm::X86ISD::SETCC_CARRY, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by combineShift().
◆ combineShiftRightArithmetic()
|
static |
Definition at line 35550 of file X86ISelLowering.cpp.
References llvm::ISD::Constant, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::APInt::isNegative(), llvm::EVT::isVector(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, Size, and llvm::ISD::SRA.
Referenced by combineShift().
◆ combineShiftRightLogical()
|
static |
Definition at line 35603 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::APInt::countTrailingOnes(), llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getMinSignedBits(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::APInt::isMask(), llvm::isPowerOf2_32(), llvm::APInt::lshr(), and llvm::ISD::SRL.
Referenced by combineShift().
◆ combineShuffle()
|
static |
Definition at line 32266 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::BITCAST, llvm::SmallVectorImpl< T >::clear(), combineShuffleOfConcatUndef(), combineShuffleToAddSubOrFMAddSub(), combineTargetShuffle(), combineX86ShufflesRecursively(), EltsFromConsecutiveLoads(), llvm::ISD::FADD, llvm::ISD::FMUL, foldShuffleOfHorizOp(), llvm::ISD::FSUB, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::ShuffleVectorSDNode::getMask(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), getShuffleScalarElt(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE41(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isOperationLegal(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::ARM_MB::LD, llvm::ISD::MUL, N, llvm::X86ISD::PMULUDQ, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::SmallVectorBase::size(), llvm::ISD::SUB, llvm::MVT::v4i32, and llvm::ISD::VECTOR_SHUFFLE.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineShuffleOfConcatUndef()
|
static |
Definition at line 32182 of file X86ISelLowering.cpp.
References Concat, llvm::ISD::CONCAT_VECTORS, llvm::MVT::f32, llvm::MVT::f64, llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i32, llvm::MVT::i64, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), N, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by combineShuffle().
◆ combineShuffleToAddSubOrFMAddSub()
|
static |
Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.
Definition at line 32146 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADDSUB, combineShuffleToFMAddSub(), llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUBADD, llvm::SelectionDAG::getNode(), llvm::SDNode::getSimpleValueType(), llvm::MVT::is512BitVector(), isAddSubOrSubAdd(), and isFMAddSubOrFMSubAdd().
Referenced by combineShuffle().
◆ combineShuffleToFMAddSub()
|
static |
Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
Definition at line 32103 of file X86ISelLowering.cpp.
References llvm::ISD::FMA, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUBADD, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDValue::hasOneUse(), isAddSubOrSubAddMask(), llvm::TargetLoweringBase::isTypeLegal(), N, std::swap(), and llvm::ISD::VECTOR_SHUFFLE.
Referenced by combineShuffleToAddSubOrFMAddSub().
◆ combineSignExtendInReg()
|
static |
Definition at line 39096 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, combineSextInRegCmov(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::ISD::isNormalLoad(), llvm::ISD::LOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::MVT::v4i32, and llvm::MVT::v4i64.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineSIntToFP()
|
static |
Definition at line 40133 of file X86ISelLowering.cpp.
References llvm::X86TargetLowering::BuildFILD(), combineVectorCompareAndMaskUnaryOp(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::MVT::f128, llvm::MVT::f16, llvm::MVT::f80, llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::X86Subtarget::getTargetLowering(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasDQI(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasX87(), llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is64Bit(), llvm::ISD::isNON_EXTLoad(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::ISD::LOAD, P, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, llvm::ISD::TRUNCATE, and llvm::X86Subtarget::useSoftFloat().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineStore()
|
static |
Definition at line 37657 of file X86ISelLowering.cpp.
References llvm::TargetLoweringBase::allowsMemoryAccess(), assert(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, detectAVGPattern(), detectAVX512SSatPattern(), detectAVX512USatPattern(), EmitTruncSStore(), extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, F(), llvm::MVT::f64, llvm::CallingConv::Fast, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::MVT::getStoreSize(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::X86Subtarget::hasAVX512(), llvm::Function::hasFnAttribute(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::integer_valuetypes(), llvm::X86Subtarget::is64Bit(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isNormalLoad(), llvm::isPowerOf2_32(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegalOrCustom(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), isVolatile(), llvm::MemSDNode::isVolatile(), llvm::MipsISD::Lo, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::MinAlign(), N, llvm::Attribute::NoImplicitFloat, llvm::SDNode::ops(), llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::TokenFactor, llvm::X86Subtarget::useSoftFloat(), llvm::MVT::v16i1, llvm::MVT::v1i1, llvm::MVT::v2i1, llvm::MVT::v32i1, llvm::MVT::v4i1, llvm::MVT::v64i1, and llvm::MVT::v8i1.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineSub()
|
static |
Definition at line 41114 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, C, combineAddOrSubToADCOrSBB(), combineIncDecVector(), combineSubToSubus(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, isHorizontalBinOp(), shouldUseHorizontalOp(), SplitOpsAndApply(), llvm::MVT::v16i16, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineSubToSubus()
|
static |
Definition at line 41026 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSSE3(), SplitOpsAndApply(), llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::X86Subtarget::useBWIRegs(), llvm::ISD::USUBSAT, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v64i8, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::MVT::v8i64.
Referenced by combineSub().
◆ combineTargetShuffle()
|
static |
Try to combine x86 target specific shuffles.
Definition at line 31746 of file X86ISelLowering.cpp.
References assert(), combineRedundantDWordShuffle(), combineX86ShufflesRecursively(), D, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FDIV, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86ISD::HADD, llvm::SDValue::hasOneUse(), llvm::MipsISD::Hi, llvm::X86ISD::HSUB, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::INSERTPS, isNoopShuffleMask(), isTargetShuffle(), llvm::SDValue::isUndef(), isUndefOrZero(), llvm::MipsISD::Lo, llvm::makeArrayRef(), llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::ISD::SCALAR_TO_VECTOR, setTargetShuffleZeroElements(), llvm::SmallVectorBase::size(), llvm::SM_SentinelUndef, std::swap(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v4f32, and llvm::X86ISD::VBROADCAST.
Referenced by combineShuffle().
◆ combineToExtendBoolVectorInReg()
|
static |
Definition at line 39256 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::SmallVectorImpl< T >::append(), assert(), llvm::tgtok::Bit, llvm::ISD::BITCAST, llvm::tgtok::Bits, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
◆ combineToExtendCMOV()
|
static |
Definition at line 39208 of file X86ISelLowering.cpp.
References llvm::X86ISD::CMOV, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, and llvm::ISD::SIGN_EXTEND.
Referenced by combineSext(), and combineZext().
◆ combineToExtendVectorInReg()
|
static |
Convert a SEXT or ZEXT of a vector to a SIGN_EXTEND_VECTOR_INREG or ZERO_EXTEND_VECTOR_INREG, this requires the splitting (or concatenating with UNDEFs) of the input to vectors of the same size as the target type which then extends the lowest elements.
Definition at line 39346 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, ExperimentalVectorWideningLegalization, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, Size, llvm::X86Subtarget::useAVX512Regs(), llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by combineSext(), and combineZext().
◆ combineTruncate()
|
static |
Definition at line 38543 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, combinePMULH(), combineTruncatedArithmetic(), combineTruncateWithSat(), combineVectorSignBitsTruncation(), combineVectorTruncation(), detectAVGPattern(), detectPMADDUBSW(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::X86ISD::MMX_MOVD2W, and llvm::MVT::x86mmx.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineTruncatedArithmetic()
|
static |
Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.
e.g. TRUNC( BINOP( X, Y ) ) –> BINOP( TRUNC( X ), TRUNC( Y ) ) TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove anything that is guaranteed to be transformed by DAGCombiner.
Definition at line 38114 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::MVT::i64, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm::EVT::isVector(), LLVM_FALLTHROUGH, llvm::ISD::MUL, llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by combineTruncate().
◆ combineTruncateWithSat()
|
static |
Definition at line 37090 of file X86ISelLowering.cpp.
References detectSSatPattern(), detectUSatPattern(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::isPowerOf2_32(), isSATValidOnAVX512Subtarget(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, truncateVectorWithPACK(), llvm::X86ISD::VTRUNCS, and llvm::X86ISD::VTRUNCUS.
Referenced by combineTruncate().
◆ combineUIntToFP()
|
static |
Definition at line 40105 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::i32, llvm::EVT::isVector(), P, llvm::SelectionDAG::SignBitIsZero(), llvm::ISD::SINT_TO_FP, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineVectorCompare()
|
static |
Definition at line 41161 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::X86ISD::PCMPEQ, and llvm::X86ISD::PCMPGT.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineVectorCompareAndMaskUnaryOp()
|
static |
Definition at line 40059 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::isVector(), and llvm::ISD::SETCC.
Referenced by combineSIntToFP().
◆ combineVectorInsert()
|
static |
Definition at line 35875 of file X86ISelLowering.cpp.
References assert(), combineX86ShufflesRecursively(), llvm::SDNode::getOpcode(), llvm::SDNode::getValueType(), llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::MVT::v16i8, and llvm::MVT::v8i16.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineVectorPack()
|
static |
Definition at line 35670 of file X86ISelLowering.cpp.
References assert(), llvm::tgtok::Bits, combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), Concat, llvm::ISD::CONCAT_VECTORS, llvm::APInt::getAllOnesValue(), getConstVector(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::APInt::getNullValue(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), llvm::APInt::isIntN(), llvm::APInt::isNegative(), llvm::SDNode::isOnlyUserOf(), llvm::APInt::isSignedIntN(), llvm::SDValue::isUndef(), llvm::SDNode::isUndef(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::APInt::setBit(), llvm::APInt::trunc(), llvm::ISD::TRUNCATE, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::MVT::v8i32, and llvm::X86ISD::VTRUNC.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineVectorShiftImm()
|
static |
Definition at line 35795 of file X86ISelLowering.cpp.
References assert(), combineX86ShufflesRecursively(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i8, llvm::ISD::isBuildVectorAllZeros(), llvm::SDNode::isOnlyUserOf(), llvm::SmallVectorBase::size(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineVectorShiftVar()
|
static |
Definition at line 35773 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::X86ISD::VSHL, llvm::X86ISD::VSRA, and llvm::X86ISD::VSRL.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineVectorSignBitsTruncation()
|
static |
This function transforms vector truncation of 'extended sign-bits' or 'extended zero-bits' values.
vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.
Definition at line 38302 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::tgtok::In, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, and truncateVectorWithPACK().
Referenced by combineTruncate().
◆ combineVectorSizedSetCCEquality()
|
static |
Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks.
Definition at line 39727 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), B, C, D, llvm::MVT::f128, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, llvm::isNullConstant(), llvm::EVT::isScalarInteger(), llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::X86Subtarget::useAVX512Regs(), llvm::MVT::v16i1, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::MVT::v32i8, X, llvm::ISD::XOR, and Y.
Referenced by combineSetCC().
◆ combineVectorTruncation()
|
static |
This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.
We do it here because after type legalization the truncation will be translated into a BUILD_VECTOR with each element that is extracted from a vector and then truncated, and it is difficult to do this optimization based on them.
Definition at line 38255 of file X86ISelLowering.cpp.
References combineVectorTruncationWithPACKSS(), combineVectorTruncationWithPACKUS(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::tgtok::In, llvm::isPowerOf2_32(), llvm::EVT::isSimple(), and llvm::EVT::isVector().
Referenced by combineTruncate().
◆ combineVectorTruncationWithPACKSS()
|
static |
Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS.
Definition at line 38239 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::tgtok::In, llvm::X86ISD::PACKSS, llvm::ISD::SIGN_EXTEND_INREG, and truncateVectorWithPACK().
Referenced by combineVectorTruncation().
◆ combineVectorTruncationWithPACKUS()
|
static |
Truncate using ISD::AND mask and X86ISD::PACKUS.
Definition at line 38205 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::tgtok::In, llvm::X86ISD::PACKUS, and truncateVectorWithPACK().
Referenced by combineVectorTruncation().
◆ combineVSelectToBLENDV()
|
static |
If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element.
This function will also call SimplfiyDemandedBits on already created BLENDV to perform additional simplifications.
Definition at line 33989 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::X86ISD::BLENDV, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::EVT::is128BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::SDNode::uses(), llvm::MVT::v32i8, and llvm::ISD::VSELECT.
Referenced by combineSelect().
◆ combineVSelectWithAllOnesOrZeros()
|
static |
If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation.
TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?
Definition at line 33823 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::ISD::OR, llvm::ISD::SETCC, std::swap(), and llvm::ISD::VSELECT.
Referenced by combineSelect().
◆ combinevXi1ConstantToInteger()
|
static |
Definition at line 32995 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i1, llvm::tgtok::In, llvm::ISD::isBuildVectorOfConstantSDNodes(), and llvm::SDValue::isUndef().
Referenced by combineBitcast(), combineCastedMaskArithmetic(), and combineStore().
◆ combineX86INT_TO_FP()
|
static |
Definition at line 38993 of file X86ISelLowering.cpp.
References llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::TargetLowering::SimplifyDemandedVectorElts().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineX86SetCC()
|
static |
Definition at line 40027 of file X86ISelLowering.cpp.
References combineSetCCEFLAGS(), llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getOperand(), and getSETCC().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineX86ShuffleChain()
|
static |
Combine an arbitrary chain of shuffles into a single instruction if possible.
This is the leaf of the recursive combine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.
Definition at line 30807 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::SmallVectorTemplateCommon< T >::data(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::end(), llvm::X86ISD::EXTRQI, llvm::X86ISD::FAND, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::APInt::getNullValue(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasFastVariableShuffle(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasVBMI(), llvm::X86Subtarget::hasVLX(), llvm::X86Subtarget::hasXOP(), llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::INSERTQI, is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), isSequentialOrUndefInRange(), isSequentialOrUndefOrZeroInRange(), llvm::TargetLoweringBase::isTypeLegal(), isUndefOrZero(), isUndefOrZeroOrInRange(), llvm::BitmaskEnumDetail::Mask(), matchBinaryPermuteVectorShuffle(), matchBinaryVectorShuffle(), matchUnaryPermuteVectorShuffle(), matchUnaryVectorShuffle(), matchVectorShuffleAsEXTRQ(), matchVectorShuffleAsINSERTQ(), llvm::peekThroughBitcasts(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::APInt::setBit(), llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::MVT::v16f32, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v64i8, llvm::MVT::v8f32, llvm::MVT::v8f64, llvm::MVT::v8i32, llvm::MVT::v8i64, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, and llvm::X86ISD::VZEXT_LOAD.
Referenced by combineX86ShufflesRecursively().
◆ combineX86ShufflesConstants()
|
static |
Definition at line 31243 of file X86ISelLowering.cpp.
References assert(), llvm::tgtok::Bits, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::APInt::getNullValue(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::MVT::isFloatingPoint(), llvm::APInt::setBit(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineX86ShufflesRecursively().
◆ combineX86ShufflesRecursively()
|
static |
Fully generic combining of x86 shuffle instructions.
This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:
1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PSHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.
Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.
FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.
Definition at line 31354 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::SDNode::areOnlyUsersOf(), assert(), llvm::ArrayRef< T >::begin(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), combineX86ShuffleChain(), combineX86ShufflesConstants(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::countTrailingZeros(), llvm::Depth, llvm::ArrayRef< T >::end(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), getZeroVector(), llvm::isPowerOf2_32(), isTargetShuffleVariableMask(), llvm::MVT::isVector(), llvm::BitmaskEnumDetail::Mask(), llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleInputs(), resolveTargetShuffleInputsAndMask(), llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and std::swap().
Referenced by combineAnd(), combineAndnp(), combineOr(), combineShuffle(), combineTargetShuffle(), combineVectorInsert(), combineVectorPack(), and combineVectorShiftImm().
◆ combineXor()
|
static |
Definition at line 38752 of file X86ISelLowering.cpp.
References combineFneg(), convertIntLogicToFPLogic(), foldVectorXorShiftIntoCmp(), foldXor1SetCC(), foldXorTruncShiftIntoCmp(), llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::MVT::v4f32, and llvm::MVT::v4i32.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ combineZext()
|
static |
Definition at line 39662 of file X86ISelLowering.cpp.
References llvm::ISD::AND, combineExtSetcc(), combineOrCmpEqZeroToCtlzSrl(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), combineToExtendVectorInReg(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isOneConstant(), llvm::EVT::isVector(), promoteExtBeforeAdd(), PromoteMaskArithmetic(), llvm::X86ISD::SETCC_CARRY, and llvm::ISD::TRUNCATE.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
◆ computeZeroableShuffleElements()
Compute whether each element of a shuffle is zeroable.
A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.
Definition at line 9726 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::APInt::getLoBits(), llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::APInt::lshrInPlace(), llvm::peekThroughBitcasts(), llvm::APInt::setBit(), Size, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerVectorShuffle().
◆ concatSubVectors()
|
static |
Definition at line 5609 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), and insertSubVector().
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ ConvertI1VectorToInteger()
|
static |
Definition at line 7756 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::tgtok::In, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), and llvm::max().
Referenced by isTruncWithZeroHighBitsInput(), and LowerBUILD_VECTORvXi1().
◆ convertIntLogicToFPLogic()
|
static |
If both input operands of a logic op are being cast from floating point types, try to convert this into a floating point logic node to avoid unnecessary moves from SSE to integer registers.
Definition at line 36100 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::ISD::DELETED_NODE, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i32, llvm::MVT::i64, llvm::EVT::isFloatingPoint(), llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by combineAnd(), combineOr(), and combineXor().
◆ convertShiftLeftToScale()
|
static |
Definition at line 24372 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, C, llvm::ISD::FP_TO_SINT, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDNode::isUndef(), llvm::MipsISD::Lo, llvm::X86ISD::PACKUS, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SHL, llvm::APInt::shl(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::Z.
Referenced by LowerRotate(), and LowerShift().
◆ CreateCopyOfByValArgument()
|
static |
Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute.
The copy will be passed as a byval function parameter.
Definition at line 2907 of file X86ISelLowering.cpp.
References llvm::ISD::ArgFlagsTy::getByValAlign(), llvm::ISD::ArgFlagsTy::getByValSize(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMemcpy(), and llvm::MVT::i32.
Referenced by getMOVL(), and isSortedByValueNo().
◆ createMMXBuildVector()
|
static |
Definition at line 33068 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasSSE1(), llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, N, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SmallVectorBase::size(), llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::Intrinsic::x86_mmx_punpcklbw, llvm::Intrinsic::x86_mmx_punpckldq, llvm::Intrinsic::x86_mmx_punpcklwd, llvm::Intrinsic::x86_sse_pshuf_w, and llvm::MVT::x86mmx.
Referenced by combineBitcast().
◆ createPackShuffleMask()
|
static |
Create a shuffle mask that matches the PACKSS/PACKUS truncation.
Note: This ignores saturation, so inputs must be checked first.
Definition at line 6030 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorBase::empty(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by getFauxShuffleMask(), and matchVectorShuffleWithPACK().
◆ createPHIsForCMOVsInSinkBB()
|
static |
Definition at line 28210 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::addDirectMem(), llvm::MachineInstrBuilder::addGlobalAddress(), llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::addLiveIn(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineInstrBuilder::addRegMask(), llvm::X86::AddrNumOperands, llvm::MachineBasicBlock::addSuccessor(), assert(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::CallingConv::C, checkAndUpdateEFLAGSKill(), llvm::classifyEHPersonality(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::N86::EAX, llvm::N86::EDI, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::erase(), llvm::MachineInstr::eraseFromParent(), llvm::N86::ESP, F(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::find(), first, llvm::MCInstrInfo::get(), llvm::MachineBasicBlock::getBasicBlock(), llvm::TargetInstrInfo::getCallFrameDestroyOpcode(), llvm::TargetInstrInfo::getCallFrameSetupOpcode(), llvm::X86RegisterInfo::getCallPreservedMask(), llvm::X86::GetCondBranchFromCond(), llvm::X86RegisterInfo::getDarwinTLSCallPreservedMask(), llvm::MachineFunction::getDataLayout(), llvm::MachineInstr::getDebugLoc(), llvm::MachineFunction::getFunction(), llvm::MachineOperand::getGlobal(), llvm::X86InstrInfo::getGlobalBaseReg(), llvm::MachineOperand::getImm(), llvm::MachineInstrBuilder::getInstr(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::X86Subtarget::getInstrInfo(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineOperand::getMBB(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::MachineBasicBlock::getParent(), llvm::Function::getPersonalityFn(), llvm::TargetLoweringBase::getPointerTy(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::TargetLoweringBase::getRegClassFor(), llvm::MachineFunction::getRegInfo(), llvm::X86Subtarget::getRegisterInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineOperand::getTargetFlags(), llvm::RegState::Implicit, llvm::RegState::ImplicitDefine, llvm::MachineBasicBlock::insert(), llvm::MachineFunction::insert(), llvm::MachineBasicBlock::insertAfter(), llvm::X86Subtarget::is32Bit(), llvm::X86Subtarget::is64Bit(), llvm::isAsynchronousEHPersonality(), isCMOVPseudo(), llvm::MachineOperand::isGlobal(), llvm::TargetLowering::isPositionIndependent(), llvm::MachineOperand::isReg(), llvm::X86Subtarget::isTarget64BitLP64(), llvm::X86Subtarget::isTargetDarwin(), llvm::X86Subtarget::isTargetNaCl64(), llvm::RegState::Kill, llvm::MachineInstr::killsRegister(), llvm_unreachable, MI, MRI, second, llvm::MachineOperand::setIsKill(), llvm::MachineOperand::setMBB(), llvm::MachineFunction::shouldSplitStack(), llvm::MachineBasicBlock::splice(), llvm::MachineBasicBlock::succ_size(), std::swap(), TII, llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs(), and TRI.
◆ createPSADBW()
|
static |
Definition at line 33384 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::MVT::i64, llvm::MVT::i8, llvm::max(), llvm::X86ISD::PSADBW, and SplitOpsAndApply().
Referenced by combineBasicSADPattern(), and combineLoopSADPattern().
◆ createShuffleMaskFromVSELECT()
|
static |
Definition at line 9652 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), llvm::SDValue::isUndef(), llvm::SmallVectorImpl< T >::resize(), Size, and llvm::SM_SentinelUndef.
Referenced by combineSelect(), and lowerVSELECTtoVectorShuffle().
◆ createTargetShuffleMask()
|
static |
Definition at line 9635 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::getBitWidth(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by lowerVectorShuffleAsBlend().
◆ createVariablePermute()
|
static |
Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices.
The vectors might be manipulated to match the type of the permute op.
Definition at line 8551 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), extract256BitVector(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelectCC(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasVBMI(), llvm::X86Subtarget::hasVLX(), llvm::X86Subtarget::hasXOP(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::isPowerOf2_64(), llvm::MipsISD::Lo, llvm::ISD::MUL, llvm::X86ISD::PSHUFB, llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::MVT::SimpleTy, SplitOpsAndApply(), llvm::MVT::v16f32, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v64i8, llvm::MVT::v8f32, llvm::MVT::v8f64, llvm::MVT::v8i16, llvm::MVT::v8i32, llvm::MVT::v8i64, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPPERM, and widenSubVector().
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ decrementVectorConstant()
|
static |
Given a simple buildvector constant, return a new vector constant with each element decremented.
If decrementing would result in underflow or this is not a simple vector constant, return an empty value.
Definition at line 19305 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), and llvm::SmallVectorTemplateBase< T >::push_back().
Referenced by LowerVSETCCWithSUBUS().
◆ detectAVGPattern()
|
static |
This function detects the AVG pattern between vectors of unsigned i8/i16, which is c = (a + b + 1) / 2, and replace this operation with the efficient X86ISD::AVG instruction.
Definition at line 37132 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::AVG, C, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i8, llvm::BuildVectorSDNode::isConstant(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::SDNode::ops(), SplitOpsAndApply(), llvm::ISD::SRL, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, llvm::APInt::ugt(), llvm::APInt::ult(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineStore(), and combineTruncate().
◆ detectAVX512SSatPattern()
|
static |
Detect a pattern of truncation with signed saturation.
The types should allow to use VPMOVSS* instruction on AVX512. Return the source value to be truncated or SDValue() if the pattern was not matched.
Definition at line 37064 of file X86ISelLowering.cpp.
References detectSSatPattern(), llvm::SDValue::getValueType(), isSATValidOnAVX512Subtarget(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by combineStore().
◆ detectAVX512USatPattern()
|
static |
Detect a pattern of truncation with saturation: (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type).
The types should allow to use VPMOVUS* instruction on AVX512. Return the source value to be truncated or SDValue() if the pattern was not matched.
Definition at line 37079 of file X86ISelLowering.cpp.
References detectUSatPattern(), llvm::SDValue::getValueType(), isSATValidOnAVX512Subtarget(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by combineStore().
◆ detectPMADDUBSW()
|
static |
Definition at line 38412 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::BUILD_VECTOR, detectSSatPattern(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::i16, llvm::MVT::i8, llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::ISD::SIGN_EXTEND, SplitOpsAndApply(), std::swap(), llvm::X86ISD::VPMADDUBSW, and llvm::ISD::ZERO_EXTEND.
Referenced by combineTruncate().
◆ detectSSatPattern()
Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type).
With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type]. Return the source value to be truncated or SDValue() if the pattern was not matched.
Definition at line 37026 of file X86ISelLowering.cpp.
References assert(), C, llvm::APInt::getAllOnesValue(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::ISD::isConstantSplatVector(), llvm::APInt::sext(), llvm::ISD::SMAX, llvm::ISD::SMIN, and llvm::APInt::zext().
Referenced by combineTruncateWithSat(), detectAVX512SSatPattern(), and detectPMADDUBSW().
◆ detectUSatPattern()
|
static |
Detect patterns of truncation with unsigned saturation:
- (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type). Return the source value x to be truncated or SDValue() if the pattern was not matched.
(truncate (smin (smax (x, C1), C2)) to dest_type), where C1 >= 0 and C2 is unsigned max of destination type.
(truncate (smax (smin (x, C2), C1)) to dest_type) where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.
These two patterns are equivalent to: (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type) So return the smax(x, C1) value to be truncated or SDValue() if the pattern was not matched.
Definition at line 36979 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isMask(), llvm::APInt::isNonNegative(), llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::APInt::uge(), and llvm::ISD::UMIN.
Referenced by combineTruncateWithSat(), and detectAVX512USatPattern().
◆ detectZextAbsDiff()
Definition at line 33319 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i8, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isOneValue(), llvm::ISD::SETCC, llvm::ISD::SETGT, llvm::ISD::SETLT, llvm::ISD::SUB, std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineBasicSADPattern(), and combineLoopSADPattern().
◆ EltsFromConsecutiveLoads()
|
static |
Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.
Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
Definition at line 7196 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::SmallBitVector::count(), llvm::count(), llvm::SmallBitVector::find_first(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MachineMemOperand::getFlags(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::ISD::isNON_EXTLoad(), llvm::MemSDNode::isNonTemporal(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::ARM_MB::LD, llvm::ISD::LOAD, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::MachineMemOperand::MOLoad, llvm::MachineMemOperand::MOVolatile, llvm::MVT::Other, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ArrayRef< T >::size(), llvm::X86ISD::VZEXT_LOAD, and llvm::Z.
Referenced by combineInsertSubvector(), combineShuffle(), and LowerBUILD_VECTORAsVariablePermute().
◆ emitClzero()
|
static |
Definition at line 27790 of file X86ISelLowering.cpp.
References llvm::ARM_AM::add, llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::X86::AddrNumOperands, llvm::MachineBasicBlock::addSuccessor(), llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, assert(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::X86::COND_AE, llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::N86::EAX, llvm::MachineBasicBlock::end(), llvm::MachineInstr::eraseFromParent(), F(), llvm::MCInstrInfo::get(), llvm::MDNode::get(), llvm::MachineBasicBlock::getBasicBlock(), llvm::Function::getCallingConv(), llvm::X86::GetCondBranchFromCond(), llvm::MachineInstr::getDebugLoc(), llvm::MachinePointerInfo::getFixedStack(), llvm::MachineFunction::getFunction(), llvm::MachineOperand::getImm(), llvm::X86Subtarget::getInstrInfo(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), getReg(), llvm::TargetLoweringBase::getRegClassFor(), llvm::MachineFunction::getRegInfo(), llvm::X86Subtarget::hasAVX(), llvm::MachineInstr::hasOneMemOperand(), llvm::MVT::i32, llvm::MVT::i64, llvm::MachineFunction::insert(), llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isCallingConvWin64(), llvm::isPowerOf2_32(), isReg(), llvm::MachineInstr::memoperands_begin(), llvm::MachineInstr::memoperands_end(), MI, llvm::MachineMemOperand::MOStore, MRI, llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
◆ EmitKORTEST()
|
static |
Definition at line 19678 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_NE, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::MVT::i8, llvm::isAllOnesConstant(), llvm::MVT::isFloatingPoint(), llvm::MVT::isInteger(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::MVT::isVector(), llvm::X86ISD::KORTEST, LowerAndToBT(), LowerVectorAllZeroTest(), LowerVSETCC(), llvm::ISD::OR, llvm::X86ISD::PTEST, llvm::X86ISD::SBB, llvm::X86ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, TranslateIntegerX86CC(), TranslateX86CC(), llvm::MVT::v16i1, llvm::MVT::v32i1, llvm::MVT::v64i1, and llvm::MVT::v8i1.
◆ EmitMaskedTruncSStore()
|
static |
Emit Masked Truncating Store with signed or unsigned saturation.
Definition at line 22611 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getTargetMemSDNode(), llvm::SelectionDAG::getVTList(), and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
◆ emitMonitor()
|
static |
Definition at line 27765 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::X86::AddrNumOperands, llvm::BuildMI(), llvm::N86::EAX, llvm::N86::ECX, llvm::N86::EDX, llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::X86Subtarget::getInstrInfo(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::X86Subtarget::is64Bit(), and TII.
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
◆ emitRDPKRU()
|
static |
Definition at line 27748 of file X86ISelLowering.cpp.
References llvm::BuildMI(), llvm::N86::EAX, llvm::N86::ECX, llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::X86Subtarget::getInstrInfo(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), and TII.
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
◆ EmitTailCallStoreRetAddr()
|
static |
Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0).
Definition at line 3553 of file X86ISelLowering.cpp.
References llvm::MachineFrameInfo::CreateFixedObject(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getStore().
Referenced by getMOVL().
◆ EmitTest()
|
static |
Emit nodes that will be selected as "test Op0,Op0", or something equivalent.
Definition at line 18861 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::X86ISD::AND, llvm::ISD::AND, assert(), llvm::X86ISD::CMP, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NO, llvm::X86::COND_O, llvm::ISD::CopyToReg, llvm::TargetLoweringBase::Enabled, llvm::MVT::f32, llvm::X86ISD::FNSTSW16r, llvm::X86ISD::FRCP, llvm::X86ISD::FRSQRT, llvm::SelectionDAG::getConstant(), llvm::SDNode::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasCMov(), llvm::X86Subtarget::hasFastScalarFSQRT(), llvm::X86Subtarget::hasFastVectorFSQRT(), llvm::X86Subtarget::hasLAHFSAHF(), hasNonFlagsUse(), llvm::SDNodeFlags::hasNoSignedWrap(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::isAtom(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), llvm::EVT::isVector(), isX86CCUnsigned(), LLVM_FALLTHROUGH, llvm_unreachable, llvm::ISD::MUL, llvm::SDNode::op_begin(), llvm::Function::optForMinSize(), llvm::X86ISD::OR, llvm::ISD::OR, llvm::X86ISD::RCP14, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::X86ISD::RSQRT14, llvm::X86ISD::SAHF, llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, llvm::ISD::STORE, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::X86Subtarget::useAVX512Regs(), llvm::MVT::v16f32, llvm::MVT::v4f32, llvm::MVT::v8f32, llvm::X86ISD::XOR, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
◆ EmitTruncSStore()
|
static |
Emit Truncating Store with signed or unsigned saturation.
Definition at line 22597 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getTargetMemSDNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::Other, and llvm::RegState::Undef.
Referenced by combineStore(), and LowerINTRINSIC_W_CHAIN().
◆ emitWRPKRU()
|
static |
Definition at line 27727 of file X86ISelLowering.cpp.
References llvm::BuildMI(), llvm::N86::EAX, llvm::N86::ECX, llvm::N86::EDX, llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::X86Subtarget::getInstrInfo(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), and TII.
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
◆ emitXBegin()
|
static |
Utility function to emit xbegin specifying the start of an RTM region.
Definition at line 27652 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::N86::EAX, llvm::MachineBasicBlock::end(), llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineBasicBlock::getBasicBlock(), llvm::MachineInstr::getDebugLoc(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::MachineFunction::getRegInfo(), I, llvm::MachineFunction::insert(), MRI, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
◆ errorUnsupported()
|
static |
Call this when the user attempts to do something unsupported, like returning a double without SSE2 enabled on x86_64.
This is not fatal, unlike report_fatal_error, so calling code should attempt to recover without crashing.
Definition at line 92 of file X86ISelLowering.cpp.
References llvm::LLVMContext::diagnose(), llvm::SelectionDAG::getContext(), llvm::SDLoc::getDebugLoc(), llvm::MachineFunction::getFunction(), and llvm::SelectionDAG::getMachineFunction().
Referenced by lowerRegToMasks(), and Passv64i1ArgInRegs().
◆ ExpandHorizontalBinOp()
|
static |
Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.
This is a helper function of LowerToHorizontalOp(). This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.
The kind of horizontal binary operation is defined by X86Opcode.
Mode specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI
Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI
If isUndefLO is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI is set, then UNDEF is propagated to the upper 128-bits of the result.
Definition at line 7996 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::AArch64CC::HI, llvm::MVT::is256BitVector(), llvm::SDNode::isUndef(), and llvm::AArch64CC::LO.
Referenced by LowerToHorizontalOp().
◆ ExtendToType()
|
static |
Widen a vector input to a vector of NVT.
The input vector must have the same element type as NVT.
Definition at line 25855 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), and llvm::SmallVectorTemplateBase< T >::push_back().
Referenced by LowerMGATHER(), LowerMLOAD(), LowerMSCATTER(), and LowerMSTORE().
◆ extract128BitVector()
|
static |
Generate a DAG to grab 128-bits from a vector > 128 bits.
This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.
Definition at line 5348 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().
Referenced by combineExtractSubvector(), combineStore(), createVariablePermute(), ExpandHorizontalBinOp(), ExtractBitFromMaskVector(), InsertBitToMaskVector(), Lower256IntVSETCC(), lowerAddSubToHorizontalOp(), LowerMULH(), LowerTruncateVecI1(), lowerVectorShuffleAsBroadcast(), and split256IntArith().
◆ extract256BitVector()
|
static |
Generate a DAG to grab 256-bits from a 512-bit vector.
Definition at line 5356 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().
Referenced by createVariablePermute(), and split512IntArith().
◆ ExtractBitFromMaskVector()
|
static |
Extract one bit from mask vector, like v16i1 or v8i1.
AVX-512 feature.
Definition at line 16306 of file X86ISelLowering.cpp.
References assert(), llvm::MipsISD::Ext, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::SDNode::isOnlyUserOf(), llvm::isPowerOf2_32(), llvm::X86ISD::KSHIFTR, LowerEXTRACT_VECTOR_ELT_SSE4(), llvm::BitmaskEnumDetail::Mask(), MayFoldIntoStore(), MayFoldIntoZeroExtend(), llvm::X86ISD::PEXTRW, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v4i32, llvm::MVT::v8i1, and llvm::MVT::v8i16.
◆ extractSubVector()
|
static |
Definition at line 5317 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), and llvm::SDNode::ops().
Referenced by combineHorizontalMinMaxResult(), createVariablePermute(), extract128BitVector(), extract256BitVector(), getExtendInVec(), getHopForBuildVector(), isTruncWithZeroHighBitsInput(), LowerEXTEND_VECTOR_INREG(), LowerVectorIntUnary(), matchUnaryVectorShuffle(), SplitOpsAndApply(), and truncateVectorWithPACK().
◆ foldShuffleOfHorizOp()
Eliminate a redundant shuffle of a horizontal math op.
Definition at line 32225 of file X86ISelLowering.cpp.
References llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::X86ISD::HADD, HOp, llvm::X86ISD::HSUB, isTargetShuffleEquivalent(), llvm::SDValue::isUndef(), N, and llvm::ISD::VECTOR_SHUFFLE.
Referenced by combineShuffle().
◆ foldVectorXorShiftIntoCmp()
|
static |
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1.
This should be called before type legalization because the pattern may not persist after that.
Definition at line 36902 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE42(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isSimple(), llvm::X86ISD::PCMPGT, llvm::MVT::SimpleTy, llvm::ISD::SRA, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by combineXor().
◆ foldXor1SetCC()
|
static |
Fold a xor(setcc cond, val), 1 –> setcc (inverted(cond), val)
Definition at line 38737 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::X86ISD::SETCC, and llvm::ISD::XOR.
Referenced by combineXor().
◆ foldXorTruncShiftIntoCmp()
|
static |
Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1)
Definition at line 36847 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isOneConstant(), llvm::ISD::SETGT, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineXor().
◆ get64BitArgumentGPRs()
|
static |
Definition at line 3092 of file X86ISelLowering.cpp.
References assert(), llvm::sys::path::begin(), llvm::sys::path::end(), llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isCallingConvWin64(), and llvm::makeArrayRef().
Referenced by isSortedByValueNo().
◆ get64BitArgumentXMMs()
|
static |
Definition at line 3110 of file X86ISelLowering.cpp.
References assert(), llvm::sys::path::begin(), llvm::sys::path::end(), F(), llvm::MachineFunction::getFunction(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isCallingConvWin64(), llvm::makeArrayRef(), llvm::Attribute::NoImplicitFloat, llvm::None, and llvm::X86Subtarget::useSoftFloat().
Referenced by isSortedByValueNo().
◆ getAVX2GatherNode()
|
static |
Definition at line 22298 of file X86ISelLowering.cpp.
References C, llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::isBuildVectorAllOnes(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
◆ getConstantVector()
|
static |
Definition at line 7353 of file X86ISelLowering.cpp.
References llvm::lltok::APFloat, assert(), llvm::APInt::extractBits(), llvm::ConstantFP::get(), llvm::ConstantVector::get(), llvm::Constant::getIntegerValue(), llvm::Type::getIntNTy(), llvm::MVT::getScalarSizeInBits(), llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T >::push_back().
Referenced by isBroadcastShuffle(), and lowerBuildVectorAsBroadcast().
◆ getConstVector() [1/2]
|
static |
Definition at line 5222 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i32, llvm::MVT::i64, llvm::TargetLoweringBase::isTypeLegal(), llvm::SmallVectorTemplateBase< T >::push_back(), and Split().
Referenced by combineVectorPack(), combineVectorShiftImm(), combineX86ShuffleChain(), combineX86ShufflesConstants(), lowerV16F32VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), and lowerVectorShuffleWithPERMV().
◆ getConstVector() [2/2]
|
static |
Definition at line 5252 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i32, llvm::MVT::i64, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::APInt::lshr(), llvm::ArrayRef< T >::size(), Split(), and llvm::APInt::trunc().
◆ getExtendedControlRegister()
|
static |
Handles the lowering of builtin intrinsic that return the value of the extended control register.
Definition at line 22406 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_PAIR, llvm::N86::EAX, llvm::N86::ECX, llvm::N86::EDX, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::AArch64CC::HI, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::AArch64CC::LO, llvm::ISD::OR, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SHL, and llvm::XGETBV.
Referenced by LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().
◆ getExtendInVec()
|
static |
Definition at line 5629 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isVector(), llvm::max(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by combineMaskedLoad(), LowerLoad(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), and llvm::X86TargetLowering::ReplaceNodeResults().
◆ getFauxShuffleMask()
|
static |
Definition at line 6476 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::SmallVectorImpl< T >::append(), assert(), llvm::ArrayRef< T >::begin(), llvm::SmallVectorImpl< T >::clear(), llvm::SelectionDAG::ComputeNumSignBits(), createPackShuffleMask(), llvm::DecodeZeroExtendMask(), llvm::ArrayRef< T >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::ISD::INSERT_SUBVECTOR, llvm::SDNode::isOnlyUserOf(), llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::EVT::isVector(), llvm::X86::isZeroNode(), llvm::BitmaskEnumDetail::Mask(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::max(), N, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleInputs(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SmallVectorBase::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::MVT::v16i8, llvm::MVT::v8i16, llvm::ISD::VECTOR_SHUFFLE, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by resolveTargetShuffleInputs().
◆ getGatherNode()
|
static |
Definition at line 22323 of file X86ISelLowering.cpp.
References C, llvm::dyn_cast(), getMaskNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::isBuildVectorAllOnes(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
◆ getHopForBuildVector()
|
static |
Definition at line 8315 of file X86ISelLowering.cpp.
References extractSubVector(), llvm::SDValue::getNode(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), and insertSubVector().
Referenced by LowerToHorizontalOp().
◆ getIndexFromUnindexedLoad()
|
static |
Definition at line 36170 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::LoadSDNode::getBasePtr(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::LSBaseSDNode::isIndexed(), and llvm::ISD::SHL.
◆ getMaskNode()
|
static |
Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics.
Definition at line 21373 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::bitsLE(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_ELEMENT, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is32Bit(), llvm::isAllOnesConstant(), llvm::X86::isZeroNode(), llvm::MipsISD::Lo, llvm::MVT::v32i1, and llvm::MVT::v64i1.
Referenced by getGatherNode(), getPrefetchNode(), getScatterNode(), getVectorMaskingNode(), LowerINTRINSIC_W_CHAIN(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPSHUFB(), and recoverFramePointer().
◆ getMaxByValAlign()
Helper for getByValTypeAlignment to determine the desired ByVal argument alignment.
Definition at line 1999 of file X86ISelLowering.cpp.
Referenced by llvm::X86TargetLowering::getByValTypeAlignment().
◆ getMOVL()
|
static |
Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width.
Definition at line 3572 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::CCValAssign::AExt, llvm::AArch64CC::AL, llvm::MachineFunction::allocateRegMask(), llvm::CCState::AllocateStack(), llvm::ISD::ANY_EXTEND, Arg, assert(), llvm::SmallVectorTemplateCommon< T >::back(), llvm::CCValAssign::BCvt, llvm::X86ISD::CALL, llvm::TargetLowering::CallLoweringInfo::CallConv, Callee, llvm::TargetLowering::CallLoweringInfo::Callee, callIsStructReturn(), canGuaranteeTCO(), llvm::TargetLowering::CallLoweringInfo::Chain, llvm::classifyEHPersonality(), llvm::X86Subtarget::classifyGlobalFunctionReference(), CreateCopyOfByValArgument(), llvm::MachineFrameInfo::CreateFixedObject(), llvm::MachineFrameInfo::CreateStackObject(), llvm::SelectionDAG::CreateStackTemporary(), llvm::TargetLowering::CallLoweringInfo::CS, llvm::TargetLowering::CallLoweringInfo::DAG, llvm::TargetLowering::CallLoweringInfo::DL, llvm::CallBase::doesNoCfCheck(), llvm::TargetLowering::CallLoweringInfo::DoesNotReturn, llvm::dyn_cast(), E, llvm::N86::EBX, EmitTailCallStoreRetAddr(), llvm::SmallVectorBase::empty(), F(), first, llvm::ISD::OutputArg::Flags, llvm::CCValAssign::Full, G, llvm::SelectionDAG::getBitcast(), llvm::X86MachineFunctionInfo::getBytesToPopOnReturn(), llvm::CallBase::getCalledFunction(), llvm::X86RegisterInfo::getCallPreservedMask(), llvm::SelectionDAG::getCALLSEQ_END(), llvm::SelectionDAG::getCALLSEQ_START(), llvm::TargetMachine::getCodeModel(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachinePointerInfo::getFixedStack(), llvm::Function::getFnAttribute(), llvm::X86MachineFunctionInfo::getForwardedMustTailRegParms(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::X86Subtarget::getFrameLowering(), llvm::MachineFunction::getFunction(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachinePointerInfo::getGOT(), llvm::MachineFunction::getInfo(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getInstruction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocMemOffset(), llvm::CCValAssign::getLocReg(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMMI(), llvm::MachineModuleInfo::getModule(), llvm::Module::getModuleFlag(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::X86RegisterInfo::getNoPreservedMask(), llvm::MCRegisterInfo::getNumRegs(), llvm::GlobalAddressSDNode::getOffset(), llvm::SDNode::getOpcode(), llvm::GlobalValue::getParent(), llvm::Function::getPersonalityFn(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getRegister(), llvm::X86Subtarget::getRegisterInfo(), llvm::SelectionDAG::getRegisterMask(), llvm::MachineOperand::getRegMaskSize(), llvm::MVT::getSizeInBits(), llvm::X86RegisterInfo::getSlotSize(), llvm::TargetFrameLowering::getStackAlignment(), llvm::SelectionDAG::getStackArgumentTokenFactor(), llvm::X86RegisterInfo::getStackRegister(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTarget(), llvm::MachineFunction::getTarget(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::TargetLoweringBase::getTargetMachine(), llvm::X86Subtarget::getTargetTriple(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::CCValAssign::getValVT(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getVTList(), llvm::ISD::GlobalAddress, llvm::X86ISD::GlobalBaseReg, llvm::MVT::Glue, llvm::TargetOptions::GuaranteedTailCallOpt, llvm::GlobalValue::hasDefaultVisibility(), llvm::CallBase::hasFnAttr(), llvm::GlobalValue::hasLocalLinkage(), llvm::Function::hasPersonalityFn(), llvm::X86Subtarget::hasSSE1(), llvm::Function::hasStructRetAttr(), I, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::CCValAssign::Indirect, llvm::MipsISD::Ins, llvm::TargetLowering::CallLoweringInfo::Ins, llvm::EVT::is128BitVector(), llvm::X86Subtarget::is64Bit(), llvm::ISD::ArgFlagsTy::isByVal(), llvm::X86::isCalleePop(), llvm::X86Subtarget::isCallingConvWin64(), llvm::isFuncletEHPersonality(), llvm::ISD::ArgFlagsTy::isInAlloca(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::isInvoke(), llvm::CCValAssign::isMemLoc(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::isMustTailCall(), llvm::Triple::isOSMSVCRT(), llvm::X86Subtarget::isPICStyleGOT(), llvm::CCValAssign::isRegLoc(), isSortedByValueNo(), llvm::TargetLowering::CallLoweringInfo::IsTailCall, llvm::X86Subtarget::isTarget64BitILP32(), llvm::X86Subtarget::isTargetMCU(), llvm::MCRegisterInfo::DiffListIterator::isValid(), llvm::TargetLowering::CallLoweringInfo::IsVarArg, llvm::EVT::isVector(), llvm::CodeModel::Large, llvm_unreachable, lowerMasksToReg(), llvm::BitmaskEnumDetail::Mask(), llvm::max(), llvm::Intrinsic::memcpy, llvm::X86II::MO_GOTPCREL, llvm::Mod, llvm::CCValAssign::needsCustom(), NotStructReturn, llvm::X86ISD::NT_CALL, llvm::TargetMachine::Options, llvm::MVT::Other, llvm::TargetLowering::CallLoweringInfo::Outs, llvm::TargetLowering::CallLoweringInfo::OutVals, Passv64i1ArgInRegs(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::report_fatal_error(), llvm::TargetLowering::CallLoweringInfo::RetTy, llvm::ISD::SCALAR_TO_VECTOR, second, llvm::MachineFrameInfo::setHasTailCall(), llvm::X86MachineFunctionInfo::setTCReturnAddrDelta(), llvm::CCValAssign::SExt, llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase::size(), llvm::Sched::Source, StackStructReturn, llvm::X86ISD::TC_RETURN, llvm::ISD::TokenFactor, llvm::TargetOptions::TrapUnreachable, TRI, llvm::Unknown, llvm::MVT::v2i64, llvm::MVT::v64i1, llvm::X86ISD::WrapperRIP, llvm::CallingConv::X86_INTR, llvm::CallingConv::X86_RegCall, llvm::CallingConv::X86_VectorCall, llvm::ISD::ZERO_EXTEND, and llvm::CCValAssign::ZExt.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ getNullFPConstForNullVal()
|
static |
If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.
In the case of a vector, the returned constant will not contain undefined elements even if the input parameter does. This makes it suitable to be used as a replacement operand with operations (eg, bitwise-and) where an undef should not propagate.
Definition at line 38821 of file X86ISelLowering.cpp.
References llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), getZeroVector(), isNullFPScalarOrVectorConst(), and llvm::EVT::isVector().
Referenced by combineFAnd(), and combineFAndn().
◆ getOnesVector()
|
static |
Returns a vector of specified type with all bits set.
Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>. Then bitcast to their original type, ensuring they get CSE'd.
Definition at line 5619 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::MVT::i32, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().
Referenced by combineExtractSubvector(), combineIncDecVector(), InsertBitToMaskVector(), and materializeVectorConstant().
◆ getOneTrueElt()
|
static |
If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.
Otherwise, return -1.
Definition at line 37298 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::i1, and llvm::SDValue::isUndef().
Referenced by getParamsForOneTrueMaskedElt().
◆ getOpcodeForRetpoline()
Definition at line 28874 of file X86ISelLowering.cpp.
References llvm_unreachable.
Referenced by getRetpolineSymbol().
◆ getPackDemandedElts()
|
static |
Definition at line 6047 of file X86ISelLowering.cpp.
References llvm::APInt::getBitWidth(), llvm::APInt::getNullValue(), llvm::EVT::getSizeInBits(), and llvm::APInt::setBit().
Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
◆ getParamsForOneTrueMaskedElt()
|
static |
Given a masked memory load/store operation, return true if it has one mask bit set.
If it has one mask bit set, then also return the memory address of the scalar element to load/store, the vector index to insert/extract that scalar element, and the alignment for the scalar memory access.
Definition at line 37333 of file X86ISelLowering.cpp.
References llvm::MemSDNode::getAlignment(), llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemoryVT(), getOneTrueElt(), llvm::EVT::getStoreSize(), llvm::EVT::getVectorElementType(), and llvm::MinAlign().
Referenced by reduceMaskedLoadToScalarLoad(), and reduceMaskedStoreToScalarStore().
◆ getPMOVMSKB()
|
static |
Definition at line 25307 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasInt256(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i8, llvm::MipsISD::Lo, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::ISD::SHL, llvm::SelectionDAG::SplitVector(), and llvm::MVT::v32i8.
Referenced by combineBitcastvxi1(), and LowerBITCAST().
◆ getPrefetchNode()
|
static |
Definition at line 22384 of file X86ISelLowering.cpp.
References C, llvm::dyn_cast(), llvm::SelectionDAG::getMachineNode(), getMaskNode(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
◆ getPSHUFShuffleMask()
|
static |
Get the PSHUF-style mask from PSHUF node.
This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.
Definition at line 31573 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.
Referenced by combineRedundantDWordShuffle(), and combineTargetShuffle().
◆ getReadPerformanceCounter()
|
static |
Handles the lowering of builtin intrinsics that read performance monitor counters (x86_rdpmc).
Definition at line 22451 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_PAIR, llvm::N86::EAX, llvm::N86::ECX, llvm::N86::EDX, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::AArch64CC::HI, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::AArch64CC::LO, llvm::ISD::OR, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::X86ISD::RDPMC_DAG, and llvm::ISD::SHL.
Referenced by LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().
◆ getReadTimeStampCounter()
|
static |
Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).
This function is also used to custom lower READCYCLECOUNTER nodes.
Definition at line 22498 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_PAIR, llvm::N86::EAX, llvm::N86::ECX, llvm::N86::EDX, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::AArch64CC::HI, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::AArch64CC::LO, llvm::ISD::OR, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::X86ISD::RDTSCP_DAG, and llvm::ISD::SHL.
Referenced by LowerINTRINSIC_W_CHAIN(), LowerREADCYCLECOUNTER(), and llvm::X86TargetLowering::ReplaceNodeResults().
◆ getRetpolineSymbol()
|
static |
Definition at line 28888 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addDisp(), llvm::addFrameReference(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addJumpTableIndex(), llvm::MachineInstrBuilder::addMBB(), llvm::X86::AddrDisp, llvm::MachineInstrBuilder::addReg(), llvm::MachineInstrBuilder::addRegMask(), llvm::addRegOffset(), llvm::X86::AddrNumOperands, llvm::MachineBasicBlock::addSuccessor(), assert(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineOperand::ChangeToES(), llvm::X86Subtarget::classifyBlockAddressReference(), llvm::MachineJumpTableInfo::createJumpTableIndex(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::RegState::Dead, llvm::N86::EAX, llvm::N86::ECX, llvm::N86::EDI, llvm::N86::EDX, llvm::MachineJumpTableInfo::EK_BlockAddress, llvm::MachineJumpTableInfo::EK_LabelDifference32, llvm::MachineBasicBlock::end(), llvm::MachineInstr::eraseFromParent(), llvm::MachineInstr::FrameSetup, llvm::MCInstrInfo::get(), llvm::X86RegisterInfo::getBaseRegister(), llvm::MachineBasicBlock::getBasicBlock(), llvm::TargetMachine::getCodeModel(), llvm::MachineFunction::getDataLayout(), llvm::MachineInstr::getDebugLoc(), llvm::MachineFunction::getFrameInfo(), llvm::X86RegisterInfo::getFrameRegister(), llvm::MachineFrameInfo::getFunctionContextIndex(), llvm::X86InstrInfo::getGlobalBaseReg(), llvm::MachineFunction::getInfo(), llvm::X86Subtarget::getInstrInfo(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::X86TargetLowering::getJumpTableEncoding(), llvm::MachineOperand::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::MachineModuleInfo::getModule(), llvm::Module::getModuleFlag(), llvm::X86RegisterInfo::getNoPreservedMask(), llvm::MachineInstr::getOpcode(), getOpcodeForRetpoline(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::TargetLoweringBase::getPointerTy(), llvm::MachineOperand::getReg(), llvm::MachineRegisterInfo::getRegClass(), llvm::TargetLoweringBase::getRegClassFor(), llvm::MachineFunction::getRegInfo(), llvm::X86InstrInfo::getRegisterInfo(), llvm::X86Subtarget::getRegisterInfo(), llvm::X86MachineFunctionInfo::getRestoreBasePointerOffset(), llvm::X86RegisterInfo::getStackRegister(), llvm::MVT::getStoreSize(), llvm::MachineFunction::getTarget(), llvm::X86RegisterInfo::hasBasePointer(), I, llvm::MVT::i32, llvm::MVT::i64, llvm::RegState::Implicit, llvm::RegState::ImplicitDefine, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::MachineFunction::insert(), llvm::X86Subtarget::is64Bit(), llvm::MachineInstr::isDebugInstr(), llvm::MachineInstr::isEHLabel(), llvm::MachineBasicBlock::isEHPad(), llvm::TargetLowering::isPositionIndependent(), llvm::MachineOperand::isReg(), llvm::X86Subtarget::isTarget64BitLP64(), llvm::X86Subtarget::isTargetNaCl64(), llvm::TargetRegisterInfo::isTypeLegalForClass(), llvm::RegState::Kill, llvm_unreachable, llvm::max(), llvm::MachineInstr::memoperands_begin(), llvm::MachineInstr::memoperands_end(), MI, MRI, llvm::MachineInstr::operands(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::MachineFunction::push_back(), Reg, llvm::MachineBasicBlock::removeSuccessor(), llvm::report_fatal_error(), llvm::reverse(), llvm::MachineInstr::setDesc(), llvm::MachineBasicBlock::setHasAddressTaken(), llvm::MachineBasicBlock::setIsEHPad(), llvm::MachineInstrBuilder::setMemRefs(), llvm::X86MachineFunctionInfo::setRestoreBasePointer(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::CodeModel::Small, llvm::MachineBasicBlock::splice(), llvm::MachineBasicBlock::succ_rbegin(), llvm::MachineBasicBlock::succ_rend(), llvm::ARMBuildAttrs::Symbol, TII, llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs(), llvm::ISD::TRAP, TRI, llvm::RegState::Undef, and llvm::X86Subtarget::useRetpolineExternalThunk().
◆ getScalarMaskingNode()
|
static |
Creates an SDNode for a predicated scalar operation.
- Returns
- (X86vselect
Mask,Op,PreservedSrc). The mask is coming as MVT::i8 and it should be transformed to MVT::v1i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction.
Definition at line 21438 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_RND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), getZeroVector(), llvm::MVT::i8, llvm::SDValue::isUndef(), llvm::X86ISD::SELECTS, llvm::MVT::v1i1, llvm::MVT::v8i1, and llvm::X86ISD::VFPCLASSS.
Referenced by recoverFramePointer().
◆ getScalarValueForVectorElement()
|
static |
Try to get a scalar value for a specific element of a vector.
Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
Definition at line 11448 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::isVector(), llvm::peekThroughBitcasts(), and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by lowerV2F64VectorShuffle(), and lowerVectorShuffleAsElementInsertion().
◆ getScatterNode()
|
static |
Definition at line 22357 of file X86ISelLowering.cpp.
References C, llvm::dyn_cast(), llvm::SelectionDAG::getMachineNode(), getMaskNode(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, llvm::BitmaskEnumDetail::Mask(), and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
◆ getSEHRegistrationNodeSize()
Definition at line 21464 of file X86ISelLowering.cpp.
References llvm::classifyEHPersonality(), llvm::Function::getPersonalityFn(), llvm::Function::hasPersonalityFn(), llvm::MSVC_CXX, llvm::MSVC_X86SEH, and llvm::report_fatal_error().
Referenced by recoverFramePointer().
◆ getSETCC()
|
static |
Helper for creating a X86ISD::SETCC node.
Definition at line 18738 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i8, and llvm::X86ISD::SETCC.
Referenced by combineCMov(), combineParity(), combineX86SetCC(), EmitKORTEST(), foldXor1SetCC(), LowerADDSUBCARRY(), LowerCMP_SWAP(), LowerINTRINSIC_W_CHAIN(), LowerXALUO(), recoverFramePointer(), and llvm::X86TargetLowering::ReplaceNodeResults().
◆ getShuffleScalarElt()
|
static |
Returns the scalar element that will make up the ith element of the result of the vector shuffle.
Definition at line 6794 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::isInteger(), isTargetShuffle(), llvm::EVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineShuffle().
◆ getShuffleVectorZeroOrUndef()
|
static |
Return a vector_shuffle of the specified vector of zero or undef vector.
This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
Definition at line 5674 of file X86ISelLowering.cpp.
References llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), and getZeroVector().
Referenced by LowerBUILD_VECTORAsVariablePermute(), and LowerUINT_TO_FP_i32().
◆ GetSplatValue()
|
static |
Definition at line 24274 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), and IsSplatVector().
Referenced by LowerScalarVariableShift().
◆ getTargetConstantBitsFromNode()
|
static |
Definition at line 5716 of file X86ISelLowering.cpp.
References llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::tgtok::Bits, llvm::SmallVectorBase::empty(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::extractBits(), llvm::Constant::getAggregateElement(), llvm::APInt::getAllOnesValue(), llvm::APInt::getBitWidth(), llvm::APInt::getBoolValue(), llvm::APInt::getNullValue(), llvm::SDValue::getOperand(), llvm::Type::getPrimitiveSizeInBits(), llvm::Type::getScalarSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), getTargetConstantFromNode(), llvm::Value::getType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), llvm::APInt::insertBits(), llvm::APInt::isAllOnesValue(), isAnyInRange(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), llvm::Type::isVectorTy(), llvm::BitmaskEnumDetail::Mask(), llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::setBit(), llvm::APInt::setBits(), llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VZEXT_MOVL.
Referenced by combineAnd(), combineBitcast(), combineVectorPack(), combineVectorShiftImm(), combineX86ShufflesConstants(), getFauxShuffleMask(), getTargetShuffleMaskIndices(), isConstantSplat(), isFNEG(), LowerRotate(), LowerVSETCC(), and setTargetShuffleZeroElements().
◆ getTargetConstantFromNode()
Definition at line 5696 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ConstantPoolSDNode::getConstVal(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SPII::Load, llvm::peekThroughBitcasts(), llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.
Referenced by getTargetConstantBitsFromNode().
◆ getTargetShuffleMask()
|
static |
Calculates the shuffle mask corresponding to the target-specific opcode.
If the mask could be calculated, returns it in Mask, returns the shuffle operands in Ops, and returns true. Sets IsUnary to true if only one source is used. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. It is an error to call this with non-empty Mask/Ops vectors.
Definition at line 6078 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), llvm::X86ISD::BLENDI, llvm::DecodeBLENDMask(), llvm::DecodeEXTRQIMask(), llvm::DecodeINSERTPSMask(), llvm::DecodeINSERTQIMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodePSLLDQMask(), llvm::DecodePSRLDQMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVectorBroadcast(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMIL2PMask(), llvm::DecodeVPERMILPMask(), llvm::DecodeVPERMMask(), llvm::DecodeVPERMV3Mask(), llvm::DecodeVPERMVMask(), llvm::DecodeVPPERMMask(), llvm::decodeVSHUF64x2FamilyMask(), llvm::DecodeZeroMoveLowMask(), llvm::SmallVectorBase::empty(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::X86ISD::EXTRQI, llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetShuffleMaskIndices(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::i8, llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, llvm_unreachable, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::SM_SentinelZero, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.
Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode(), getPSHUFShuffleMask(), getShuffleScalarElt(), setTargetShuffleZeroElements(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), and XFormVExtractWithShuffleIntoLoad().
◆ getTargetShuffleMaskIndices()
|
static |
Definition at line 6010 of file X86ISelLowering.cpp.
References getTargetConstantBitsFromNode(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by getTargetShuffleMask().
◆ getTargetVShiftByConstNode()
|
static |
Handle vector element shifts where the shift amount is a constant.
Takes immediate version of shift as input.
Definition at line 21222 of file X86ISelLowering.cpp.
References assert(), C, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::i8, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDNode::isUndef(), llvm_unreachable, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.
Referenced by getTargetVShiftNode(), LowerMUL(), LowerMULH(), LowerScalarImmediateShift(), LowerScalarVariableShift(), and LowerShift().
◆ getTargetVShiftNode()
|
static |
Handle vector element shifts where the shift amount may or may not be a constant.
Takes immediate version of shift as input.
Definition at line 21303 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by LowerScalarVariableShift(), and recoverFramePointer().
◆ getTargetVShiftUniformOpcode()
Definition at line 21202 of file X86ISelLowering.cpp.
References llvm_unreachable, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, and llvm::X86ISD::VSRLI.
Referenced by getTargetVShiftNode(), LowerScalarImmediateShift(), LowerScalarVariableShift(), and LowerShift().
◆ GetTLSADDR()
|
static |
Definition at line 16902 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getCopyFromReg(), llvm::MachineFunction::getFrameInfo(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::TLSModel::LocalDynamic, llvm::MVT::Other, llvm::MachineFrameInfo::setAdjustsStack(), llvm::MachineFrameInfo::setHasCalls(), llvm::X86ISD::TLSADDR, and llvm::X86ISD::TLSBASEADDR.
Referenced by LowerToTLSGeneralDynamicModel32(), LowerToTLSGeneralDynamicModel64(), and LowerToTLSLocalDynamicModel().
◆ getUnderlyingExtractedFromVec()
For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.
Modifies ExtractedFromVec to the real vector and returns the real index.
Definition at line 7654 of file X86ISelLowering.cpp.
References assert(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), and isUndefOrInRange().
Referenced by buildFromShuffleMostly().
◆ getUnpackh()
|
static |
Returns a vector_shuffle node for an unpackh operation.
Definition at line 5663 of file X86ISelLowering.cpp.
References llvm::createUnpackShuffleMask(), llvm::SelectionDAG::getVectorShuffle(), and llvm::BitmaskEnumDetail::Mask().
Referenced by convertShiftLeftToScale(), LowerAVXExtend(), LowerHorizontalByteSum(), LowerMUL(), LowerMULH(), LowerShift(), and llvm::X86TargetLowering::ReplaceNodeResults().
◆ getUnpackl()
|
static |
Returns a vector_shuffle node for an unpackl operation.
Definition at line 5655 of file X86ISelLowering.cpp.
References llvm::createUnpackShuffleMask(), llvm::SelectionDAG::getVectorShuffle(), and llvm::BitmaskEnumDetail::Mask().
Referenced by convertShiftLeftToScale(), LowerBUILD_VECTORAsVariablePermute(), LowerHorizontalByteSum(), LowerMUL(), LowerMULH(), LowerShift(), LowerUINT_TO_FP_i64(), reduceVMULWidth(), and llvm::X86TargetLowering::ReplaceNodeResults().
◆ getV4X86ShuffleImm()
Get a 4-lane 8-bit shuffle immediate for a mask.
This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.
NB: We rely heavily on "undef" masks preserving the input lane.
Definition at line 9698 of file X86ISelLowering.cpp.
References assert(), and llvm::ArrayRef< T >::size().
Referenced by getV4X86ShuffleImm8ForMask(), matchBinaryPermuteVectorShuffle(), and matchUnaryPermuteVectorShuffle().
◆ getV4X86ShuffleImm8ForMask()
|
static |
Definition at line 9713 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), getV4X86ShuffleImm(), and llvm::MVT::i8.
Referenced by combineRedundantDWordShuffle(), combineTargetShuffle(), lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerV8I32VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), and lowerVectorShuffleWithSHUFPS().
◆ getv64i1Argument()
|
static |
Reads two 32 bit registers and creates a 64 bit mask value.
- Parameters
-
VA The current 32 bit value that need to be assigned. NextVA The next 32 bit value that need to be assigned. Root The parent DAG node. [in,out] InFlag Represents SDvalue in the parent DAG node for glue purposes. In the case the DAG is already using physical register instead of virtual, we should glue our new SDValue to InFlag SDvalue.
- Returns
- a new SDvalue of size 64bit.
Definition at line 2692 of file X86ISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCopyFromReg(), llvm::CCValAssign::getLocReg(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::CCValAssign::getValVT(), llvm::X86Subtarget::hasBWI(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::X86Subtarget::is32Bit(), llvm::CCValAssign::isRegLoc(), llvm::MipsISD::Lo, Reg, llvm::MVT::v32i1, and llvm::MVT::v64i1.
Referenced by isSortedByValueNo(), and lowerRegToMasks().
◆ getVectorMaskingNode()
|
static |
Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics.
Definition at line 21412 of file X86ISelLowering.cpp.
References getMaskNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i1, llvm::isAllOnesConstant(), llvm::SDValue::isUndef(), and llvm::ISD::VSELECT.
Referenced by lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBlend(), and recoverFramePointer().
◆ getVShift()
|
static |
Return a vector logical shift node.
Definition at line 7106 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i8, llvm::EVT::is128BitVector(), llvm::MVT::v16i8, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ getX86XALUOOp()
|
static |
Definition at line 19823 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, assert(), llvm::X86::COND_B, llvm::X86::COND_O, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm_unreachable, llvm::ISD::SADDO, llvm::X86ISD::SMUL, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::X86ISD::SUB, llvm::ISD::UADDO, llvm::X86ISD::UMUL, llvm::ISD::UMULO, and llvm::ISD::USUBO.
Referenced by isTruncWithZeroHighBitsInput(), isXor1OfSetCC(), and LowerXALUO().
◆ getZeroVector()
|
static |
Returns a vector of specified type with all zero elements.
Definition at line 5294 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), and llvm::MVT::v4f32.
Referenced by combineExtractSubvector(), combineInsertSubvector(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), createVariablePermute(), getAVX2GatherNode(), getGatherNode(), getNullFPConstForNullVal(), getScalarMaskingNode(), getShuffleVectorZeroOrUndef(), getVectorMaskingNode(), insert1BitVector(), InsertBitToMaskVector(), lower1BitVectorShuffle(), LowerAVXCONCAT_VECTORS(), LowerBUILD_VECTORAsVariablePermute(), LowerBuildVectorAsInsert(), LowerBuildVectorv16i8(), LowerBuildVectorv4x32(), LowerCONCAT_VECTORSvXi1(), LowerSCALAR_TO_VECTOR(), lowerV16I8VectorShuffle(), lowerV2X128VectorShuffle(), lowerV4X128VectorShuffle(), lowerVectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), lowerVectorShuffleToEXPAND(), matchBinaryPermuteVectorShuffle(), matchVectorShuffleWithUNPCK(), materializeVectorConstant(), recoverFramePointer(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), and widenSubVector().
◆ hasBZHI()
|
static |
Definition at line 36188 of file X86ISelLowering.cpp.
References llvm::MVT::getSizeInBits(), llvm::X86Subtarget::hasBMI2(), llvm::X86Subtarget::is64Bit(), llvm::MVT::isScalarInteger(), and true.
◆ hasFPCMov()
Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
Definition at line 4764 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.
Referenced by combineCMov(), and isTruncWithZeroHighBitsInput().
◆ hasNonFlagsUse()
return true if Op has a use that doesn't just read flags.
Definition at line 18841 of file X86ISelLowering.cpp.
References llvm::ISD::BRCOND, llvm::SDNode::getOpcode(), llvm::SDNode::use_iterator::getOperandNo(), llvm::SDNode::hasOneUse(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
Referenced by EmitTest().
◆ hasStackGuardSlotTLS()
Definition at line 2244 of file X86ISelLowering.cpp.
References llvm::Triple::isAndroid(), llvm::Triple::isAndroidVersionLT(), llvm::Triple::isOSFuchsia(), and llvm::Triple::isOSGlibc().
Referenced by llvm::X86TargetLowering::getIRStackGuard(), and llvm::X86TargetLowering::insertSSPDeclarations().
◆ insert128BitVector()
|
static |
Generate a DAG to put 128-bits into a vector > 128 bits.
This sets things up to match to an AVX VINSERTF128/VINSERTI128 or AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a simple superregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering INSERT_VECTOR_ELT operations easier.
Definition at line 5392 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getValueType(), insertSubVector(), and llvm::EVT::is128BitVector().
Referenced by InsertBitToMaskVector(), LowerBUILD_VECTORAsVariablePermute(), LowerSCALAR_TO_VECTOR(), and lowerV4X128VectorShuffle().
◆ insert1BitVector()
|
static |
Insert i1-subvector to i1-vector.
Definition at line 5474 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasDQI(), llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm::X86ISD::KSHIFTL, llvm::X86ISD::KSHIFTR, llvm::ISD::OR, llvm::RegState::Undef, llvm::MVT::v16i1, llvm::MVT::v8i1, and llvm::ISD::XOR.
Referenced by LowerINSERT_SUBVECTOR().
◆ InsertBitToMaskVector()
|
static |
Insert one bit to mask vector, like v16i1 or v8i1.
AVX-512 feature.
Definition at line 16499 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::X86ISD::BLENDI, extract128BitVector(), llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, insert128BitVector(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::isAllOnesConstant(), llvm::MVT::isInteger(), llvm::isPowerOf2_32(), llvm::X86::isZeroNode(), MayFoldLoad(), llvm::Function::optForMinSize(), llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::MVT::v16i8, llvm::MVT::v1i1, llvm::MVT::v4f32, and llvm::MVT::v8i16.
◆ insertSubVector()
|
static |
Definition at line 5362 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::isPowerOf2_32(), and llvm::SDValue::isUndef().
Referenced by concatSubVectors(), getHopForBuildVector(), and insert128BitVector().
◆ is128BitLaneCrossingShuffleMask()
Test whether there are elements crossing 128-bit lanes in this shuffle mask.
X86 divides up its shuffles into in-lane and cross-lane shuffle operations and we routinely test for these.
Definition at line 9467 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), Size, and llvm::ArrayRef< T >::size().
Referenced by combineX86ShuffleChain(), llvm::X86TargetLowering::isDesirableToCombineBuildVectorToShuffleTruncate(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerVectorShuffleAsBlendOfPSHUFBs(), lowerVectorShuffleAsByteRotateAndPermute(), and matchUnaryPermuteVectorShuffle().
◆ is128BitLaneRepeatedShuffleMask() [1/2]
|
static |
Test whether a shuffle mask is equivalent within each 128-bit lane.
Definition at line 9517 of file X86ISelLowering.cpp.
References isRepeatedShuffleMask().
Referenced by LowerShift(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleByMerging128BitLanes(), matchUnaryPermuteVectorShuffle(), and matchVectorShuffleAsByteRotate().
◆ is128BitLaneRepeatedShuffleMask() [2/2]
Definition at line 9523 of file X86ISelLowering.cpp.
References isRepeatedShuffleMask().
◆ is256BitLaneRepeatedShuffleMask()
|
static |
Test whether a shuffle mask is equivalent within each 256-bit lane.
Definition at line 9530 of file X86ISelLowering.cpp.
References isRepeatedShuffleMask().
Referenced by lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), and matchUnaryPermuteVectorShuffle().
◆ isAddSubOrSubAdd() [1/2]
|
static |
Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation.
If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1 (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters Opnd0 and Opnd1.
Definition at line 8037 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE3(), llvm::MVT::isFloatingPoint(), llvm::SDValue::isUndef(), std::swap(), and llvm::ISD::UNDEF.
Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().
◆ isAddSubOrSubAdd() [2/2]
|
static |
Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation.
If true is returned then the operands of ADDSUB(SUBADD) operation are written to the parameters Opnd0 and Opnd1.
We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes so it is easier to generically match. We also insert dummy vector shuffle nodes for the operands which explicitly discard the lanes which are unused by this operation to try to flow through the rest of the combiner the fact that they're unused.
Definition at line 32043 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE3(), isAddSubOrSubAddMask(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::NVPTX::PTXLdStInstCode::V2, and llvm::ISD::VECTOR_SHUFFLE.
◆ isAddSubOrSubAddMask()
Checks if the shuffle mask takes subsequent elements alternately from two vectors.
For example <0, 5, 2, 7> or <8, 1, 10, 3, 12, 5, 14, 7> are both correct.
Definition at line 32006 of file X86ISelLowering.cpp.
References Size, and llvm::ArrayRef< T >::size().
Referenced by combineShuffleToFMAddSub(), and isAddSubOrSubAdd().
◆ isAndOrOfSetCCs()
Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.
Definition at line 20731 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::X86ISD::SETCC.
Referenced by combineCompareEqual(), and isXor1OfSetCC().
◆ isAnyInRange()
Return true if the value of any element in Mask falls within the specified range (L, H].
Definition at line 5067 of file X86ISelLowering.cpp.
References isInRange().
Referenced by getTargetConstantBitsFromNode(), and matchVectorShuffleAsVPMOV().
◆ isCMOVPseudo()
|
static |
Definition at line 28177 of file X86ISelLowering.cpp.
References llvm::MachineInstr::getOpcode().
Referenced by createPHIsForCMOVsInSinkBB().
◆ isConstantSplat()
Definition at line 5986 of file X86ISelLowering.cpp.
References llvm::SDValue::getScalarValueSizeInBits(), and getTargetConstantBitsFromNode().
Referenced by combineIncDecVector(), LowerFunnelShift(), LowerScalarImmediateShift(), and LowerVSETCC().
◆ isExpandWithZeros()
Definition at line 9293 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), and llvm::ISD::isBuildVectorAllZeros().
Referenced by isTypePromotionOfi1ZeroUpBits().
◆ isFMAddSubOrFMSubAdd()
|
static |
Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1).
If (and only if) true is returned, the operands of FMADDSUB/FMSUBADD are written to parameters Opnd0, Opnd1, Opnd2.
Prior to calling this function it should be known that there is some SDNode that potentially can be replaced with an X86ISD::ADDSUB operation using Opnd0 and Opnd1 as operands. Also, this method is called before replacement of such SDNode with ADDSUB operation. Thus the number of Opnd0 uses is expected to be equal to 2. For example, this function may be called for the following IR: AB = fmul fast <2 x double> A, B Sub = fsub fast <2 x double> AB, C Add = fadd fast <2 x double> AB, C Addsub = shufflevector <2 x double> Sub, <2 x double> Add, <2 x i32> <i32 0, i32 3> There is a def for Addsub here, which potentially can be replaced by X86ISD::ADDSUB operation: Addsub = X86ISD::ADDSUB AB, C and such ADDSUB can further be replaced with FMADDSUB: Addsub = FMADDSUB A, B, C.
The main reason why this method is called before the replacement of the recognized ADDSUB idiom with ADDSUB operation is that such replacement is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit FMADDSUB is.
Definition at line 8165 of file X86ISelLowering.cpp.
References llvm::TargetOptions::AllowFPOpFusion, llvm::FPOpFusion::Fast, llvm::ISD::FMUL, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTarget(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNode::hasNUsesOfValue(), llvm::TargetMachine::Options, and llvm::TargetOptions::UnsafeFPMath.
Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().
◆ isFNEG()
|
static |
Returns the negated value if the node N flips sign of FP value.
FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000) or FSUB(0, x) AVX512F does not have FXOR, so FNEG is lowered as (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))). In this case we go though all bitcasts. This also recognizes splat of a negated value and returns the splat of that value.
Definition at line 38593 of file X86ISelLowering.cpp.
References E, llvm::ISD::FNEG, llvm::ISD::FSUB, llvm::X86ISD::FXOR, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), llvm::SDValue::isUndef(), llvm::peekThroughBitcasts(), std::swap(), and llvm::ISD::XOR.
Referenced by combineFMA(), combineFMADDSUB(), and combineFneg().
◆ isFRClass()
|
static |
Check if RC is a vector register class.
I.e., FR* / VR* or one of their variant.
Definition at line 42316 of file X86ISelLowering.cpp.
References llvm::TargetRegisterClass::hasSuperClassEq().
Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().
◆ isGRClass()
|
static |
Check if RC is a general purpose register class.
I.e., GR* or one of their variant.
Definition at line 42306 of file X86ISelLowering.cpp.
References llvm::TargetRegisterClass::hasSuperClassEq().
Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().
◆ isHopBuildVector()
|
static |
Definition at line 8228 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::DELETED_NODE, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::SDValue::hasOneUse(), llvm::X86ISD::HSUB, llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), and llvm::ISD::SUB.
Referenced by LowerToHorizontalOp().
◆ isHorizontalBinOp()
Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.
A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'.
Definition at line 37979 of file X86ISelLowering.cpp.
References assert(), B, llvm::SmallVectorTemplateCommon< T >::begin(), C, llvm::ShuffleVectorSDNode::commuteMask(), llvm::copy(), D, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), std::swap(), and llvm::ISD::VECTOR_SHUFFLE.
Referenced by combineAdd(), combineFaddFsub(), and combineSub().
◆ isHorizontalBinOpPart()
|
static |
This is a helper function of LowerToHorizontalOp().
This function checks that the build_vector N in input implements a 128-bit partial horizontal operation on a 256-bit vector, but that operation may not match the layout of an x86 256-bit horizontal instruction. In other words, if this returns true, then some extraction/insertion will be required to produce a valid horizontal instruction.
Parameter Opcode defines the kind of horizontal operation to match. For example, if Opcode is equal to ISD::ADD, then this function checks if N implements a horizontal arithmetic add; if instead Opcode is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.
This function only analyzes elements of N whose indices are in range [BaseIdx, LastIdx).
TODO: This function was originally used to match both real and fake partial horizontal operations, but the index-matching logic is incorrect for that. See the corrected implementation in isHopBuildVector(). Can we reduce this code because it is only used for partial h-op matching now?
Definition at line 7882 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::EVT::is256BitVector(), llvm::SDValue::isUndef(), llvm::SDNode::isUndef(), and llvm::EVT::isVector().
Referenced by LowerToHorizontalOp().
◆ isInRange()
|
static |
Return true if Val falls within the specified range (L, H].
Definition at line 5061 of file X86ISelLowering.cpp.
Referenced by isAnyInRange(), isUndefOrInRange(), and isUndefOrZeroOrInRange().
◆ isMaskedZeroUpperBitsvXi1()
Definition at line 5462 of file X86ISelLowering.cpp.
References llvm::X86ISD::CMPM, llvm::X86ISD::CMPM_RND, and llvm::ISD::SETCC.
Referenced by isTypePromotionOfi1ZeroUpBits().
◆ isNonZeroElementsInOrder()
|
static |
Definition at line 9803 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::getVectorNumElements(), and llvm::ArrayRef< T >::size().
Referenced by lowerVectorShuffleToEXPAND().
◆ isNoopShuffleMask()
Tiny helper function to identify a no-op mask.
This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.
Definition at line 9453 of file X86ISelLowering.cpp.
References assert(), Size, and llvm::ArrayRef< T >::size().
Referenced by combineTargetShuffle(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), and lowerVectorShuffleAsPermuteAndUnpack().
◆ isNullFPScalarOrVectorConst()
Definition at line 38811 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::ISD::isBuildVectorAllZeros(), and llvm::isNullFPConstant().
Referenced by combineFAndn(), combineFOr(), and getNullFPConstForNullVal().
◆ isRepeatedShuffleMask()
|
static |
Test whether a shuffle mask is equivalent within each sub-lane.
This checks a shuffle mask to see if it is performing the same lane-relative shuffle in each sub-lane. This trivially implies that it is also not lane-crossing. It may however involve a blend from the same lane of a second vector.
The specific repeated shuffle mask is populated in RepeatedMask, as it is non-trivial to compute in the face of undef lanes. The representation is suitable for use with existing 128-bit shuffles as entries from the second vector have been remapped to [LaneSize, 2*LaneSize).
Definition at line 9487 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), Size, llvm::ArrayRef< T >::size(), and llvm::SM_SentinelUndef.
Referenced by is128BitLaneRepeatedShuffleMask(), and is256BitLaneRepeatedShuffleMask().
◆ isRepeatedTargetShuffleMask()
|
static |
Test whether a target shuffle mask is equivalent within each sub-lane.
Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
Definition at line 9537 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), isUndefOrZero(), Size, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by matchBinaryPermuteVectorShuffle().
◆ isSATValidOnAVX512Subtarget()
|
static |
Check if truncation with saturation form type SrcVT to DstVT is valid for the given Subtarget.
Definition at line 36945 of file X86ISelLowering.cpp.
References llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::EVT::is512BitVector(), and llvm::EVT::isVector().
Referenced by combineTruncateWithSat(), detectAVX512SSatPattern(), and detectAVX512USatPattern().
◆ isSequentialOrUndefInRange()
|
static |
Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
Definition at line 5108 of file X86ISelLowering.cpp.
References isUndefOrEqual().
Referenced by combineX86ShuffleChain(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsLanePermuteAndPermute(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithUndefHalf(), match1BitShuffleAsKSHIFT(), matchUnaryPermuteVectorShuffle(), matchVectorShuffleAsINSERTQ(), matchVectorShuffleAsShift(), matchVectorShuffleAsVPMOV(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
◆ isSequentialOrUndefOrZeroInRange()
|
static |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.
Definition at line 5119 of file X86ISelLowering.cpp.
References isUndefOrZero().
Referenced by combineX86ShuffleChain(), and matchVectorShuffleWithUNPCK().
◆ isShuffleEquivalent()
|
static |
Checks whether a shuffle mask is equivalent to an explicit list of arguments.
This is a fast way to test a shuffle mask against a fixed pattern:
if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }
It returns true if the mask is exactly as wide as the argument list, and each element of the mask is either -1 (signifying undef) or the value given in the argument.
Definition at line 9581 of file X86ISelLowering.cpp.
References assert(), llvm::dyn_cast(), Size, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16F32VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4X128VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerVectorShuffleAsBroadcast(), and lowerVectorShuffleWithUNPCK().
◆ isShuffleFoldableLoad()
Helper to test for a load that can be folded with x86 shuffles.
This is particularly important because the set of instructions varies significantly based on whether the operand is a load or not.
Definition at line 11476 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::ISD::isNON_EXTLoad(), and llvm::peekThroughBitcasts().
Referenced by lowerVectorShuffleAsBroadcast().
◆ isShuffleMaskInputInPlace()
Test whether the specified input (0 or 1) is in-place blended by the given mask.
This returns true if the elements from a particular input are already in the slot required by the given mask and require no permutation.
Definition at line 14392 of file X86ISelLowering.cpp.
References assert(), Size, and llvm::ArrayRef< T >::size().
Referenced by lowerV4F64VectorShuffle(), and lowerV4I64VectorShuffle().
◆ isSingleSHUFPSMask()
Test whether this can be lowered with a single SHUFPS instruction.
This is used to disable more specialized lowerings when the shufps lowering will happen to be efficient.
Definition at line 12259 of file X86ISelLowering.cpp.
References assert(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I32VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I32VectorShuffle().
◆ isSortedByValueNo()
|
static |
Definition at line 3140 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::MachineFunction::addLiveIn(), llvm::AArch64CC::AL, llvm::CCState::AllocateStack(), Arg, ArgGPRs, argsAreStructReturn(), assert(), llvm::ISD::AssertSext, llvm::ISD::AssertZext, B, llvm::CCValAssign::BCvt, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ArrayRef< T >::begin(), canGuaranteeTCO(), llvm::classifyEHPersonality(), llvm::CoreCLR, CreateCopyOfByValArgument(), llvm::MachineFrameInfo::CreateFixedObject(), llvm::MachineFrameInfo::CreateStackObject(), llvm::MachineRegisterInfo::createVirtualRegister(), llvm::MachineRegisterInfo::disableCalleeSavedRegister(), E, llvm::SmallVectorBase::empty(), llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::ArrayRef< T >::end(), F(), llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::f80, get64BitArgumentGPRs(), get64BitArgumentXMMs(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachinePointerInfo::getFixedStack(), llvm::X86MachineFunctionInfo::getForwardedMustTailRegParms(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::X86Subtarget::getFrameLowering(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::CCValAssign::getLocMemOffset(), llvm::SelectionDAG::getMachineFunction(), llvm::Value::getName(), llvm::SelectionDAG::getNode(), llvm::Function::getPersonalityFn(), llvm::TargetLoweringBase::getPointerTy(), llvm::TargetLoweringBase::getRegClassFor(), llvm::MachineFunction::getRegInfo(), llvm::X86MachineFunctionInfo::getRegSaveFrameIndex(), llvm::X86TargetLowering::getReturnAddressFrameIndex(), llvm::X86MachineFunctionInfo::getSRetReturnReg(), llvm::MachinePointerInfo::getStack(), llvm::SelectionDAG::getStore(), llvm::MachineFunction::getTarget(), llvm::X86Subtarget::getTargetTriple(), getv64i1Argument(), llvm::CCValAssign::getValNo(), llvm::SelectionDAG::getValueType(), llvm::X86MachineFunctionInfo::getVarArgsFPOffset(), llvm::X86MachineFunctionInfo::getVarArgsGPOffset(), llvm::MachineFunction::getWinEHFuncInfo(), llvm::GPR, llvm::TargetOptions::GuaranteedTailCallOpt, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::GlobalValue::hasExternalLinkage(), llvm::Function::hasFnAttribute(), llvm::MachineFrameInfo::hasMustTailInVarArgFunc(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MachineFrameInfo::hasVAStart(), llvm::X86Subtarget::hasVLX(), I, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::CCValAssign::Indirect, llvm::SmallVectorImpl< T >::insert(), llvm::CallingConv::Intel_OCL_BI, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::X86Subtarget::is64Bit(), llvm::ISD::ArgFlagsTy::isByVal(), llvm::X86::isCalleePop(), llvm::X86Subtarget::isCallingConvWin64(), llvm::Triple::isOSMSVCRT(), llvm::X86Subtarget::isTargetCygMing(), llvm::X86Subtarget::isTargetMCU(), llvm::EVT::isVector(), llvm::MachineRegisterInfo::liveins(), llvm_unreachable, lowerRegToMasks(), llvm::X86ISD::MOVDQ2Q, MRI, llvm::Attribute::NoImplicitFloat, llvm::TargetMachine::Options, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), Reg, llvm::report_fatal_error(), llvm::X86MachineFunctionInfo::setArgumentStackSize(), llvm::X86MachineFunctionInfo::setBytesToPopOnReturn(), llvm::X86MachineFunctionInfo::setForceFramePointer(), llvm::X86MachineFunctionInfo::setRegSaveFrameIndex(), llvm::X86MachineFunctionInfo::setSRetReturnReg(), llvm::X86MachineFunctionInfo::setVarArgsFPOffset(), llvm::X86MachineFunctionInfo::setVarArgsFrameIndex(), llvm::X86MachineFunctionInfo::setVarArgsGPOffset(), llvm::CCValAssign::SExt, shouldGuaranteeTCO(), llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::ArrayRef< T >::slice(), StackStructReturn, llvm::SPII::Store, llvm::CallingConv::Swift, llvm::ISD::TokenFactor, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useSoftFloat(), llvm::MVT::v16f32, llvm::MVT::v16i1, llvm::MVT::v1i1, llvm::MVT::v32i1, llvm::MVT::v4f32, llvm::MVT::v64i1, llvm::MVT::v8f32, llvm::MVT::v8i1, llvm::X86ISD::VASTART_SAVE_XMM_REGS, llvm::CallingConv::X86_FastCall, llvm::CallingConv::X86_INTR, llvm::CallingConv::X86_RegCall, llvm::CallingConv::X86_ThisCall, llvm::CallingConv::X86_VectorCall, llvm::MVT::x86mmx, and llvm::CCValAssign::ZExt.
Referenced by getMOVL().
◆ IsSplatVector()
|
static |
Definition at line 24237 of file X86ISelLowering.cpp.
References llvm::countTrailingOnes(), llvm::APInt::getAllOnesValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::isSplatValue(), llvm::APInt::isSubsetOf(), peekThroughEXTRACT_SUBVECTORs(), and llvm::ISD::VECTOR_SHUFFLE.
Referenced by GetSplatValue().
◆ isSplatZeroExtended()
|
static |
Definition at line 7391 of file X86ISelLowering.cpp.
References llvm::MVT::getIntegerVT(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDNode::getSimpleValueType(), llvm::isNullConstant(), llvm::isPowerOf2_32(), and llvm::SDValue::isUndef().
Referenced by lowerBuildVectorAsBroadcast().
◆ isTargetShuffle()
Definition at line 4522 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDI, llvm::X86ISD::EXTRQI, llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.
Referenced by combineShuffle(), combineTargetShuffle(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), getShuffleScalarElt(), isUseOfShuffle(), setTargetShuffleZeroElements(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), and XFormVExtractWithShuffleIntoLoad().
◆ isTargetShuffleEquivalent()
Checks whether a target shuffle mask is equivalent to an explicit pattern.
The masks must be exactly the same width.
If an element in Mask matches SM_SentinelUndef (-1) then the corresponding value in ExpectedMask is always accepted. Otherwise the indices must match.
SM_SentinelZero is accepted as a valid negative index but must match in both.
Definition at line 9616 of file X86ISelLowering.cpp.
References Size, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by foldShuffleOfHorizOp(), isUnpackWdShuffleMask(), matchBinaryVectorShuffle(), matchUnaryVectorShuffle(), matchVectorShuffleWithPACK(), and matchVectorShuffleWithUNPCK().
◆ isTargetShuffleVariableMask()
Definition at line 4561 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, and llvm::X86ISD::VPPERM.
Referenced by combineX86ShufflesRecursively().
◆ isTruncWithZeroHighBitsInput()
|
static |
Definition at line 19897 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, llvm::tgtok::Bits, llvm::X86ISD::BT, llvm::X86ISD::CMOV, llvm::X86ISD::CMP, llvm::ISD::CONCAT_VECTORS, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, ConvertI1VectorToInteger(), llvm::ISD::CopyFromReg, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::X86ISD::FOR, llvm::X86ISD::FSETCC, llvm::X86ISD::FSETCCM, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), getX86XALUOOp(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasCMov(), hasFPCMov(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::isAllOnesConstant(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MVT::isFloatingPoint(), llvm::isNullConstant(), llvm::isNullFPConstant(), llvm::isOneConstant(), llvm::X86TargetLowering::isScalarFPTypeInSSEReg(), llvm::MVT::isVector(), isX86LogicalCmp(), llvm::MipsISD::Lo, LowerAndToBT(), llvm::SelectionDAG::MaskedValueIsZero(), MayFoldLoad(), llvm::ISD::OR, llvm::ISD::SADDO, llvm::X86ISD::SBB, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SELECTS, llvm::X86ISD::SETCC, llvm::ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SETNE, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::ISD::SUB, llvm::X86ISD::SUB, T1, translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::ISD::UMULO, llvm::ISD::USUBO, llvm::MVT::v1i1, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v32i1, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v64i1, llvm::MVT::v8i1, llvm::ISD::XOR, and Y.
Referenced by isXor1OfSetCC().
◆ isTypePromotionOfi1ZeroUpBits()
Definition at line 9307 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::CONCAT_VECTORS, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), isExpandWithZeros(), and isMaskedZeroUpperBitsvXi1().
Referenced by LowerCONCAT_VECTORSvXi1().
◆ isUndefInRange()
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.
Definition at line 5053 of file X86ISelLowering.cpp.
References llvm::SM_SentinelUndef.
Referenced by lowerVectorShuffleAsSpecificZeroOrAnyExtend(), lowerVectorShuffleWithUndefHalf(), matchBinaryPermuteVectorShuffle(), matchUnaryVectorShuffle(), matchVectorShuffleAsEXTRQ(), matchVectorShuffleAsINSERTQ(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
◆ isUndefOrEqual()
|
static |
Val is the undef sentinel value or equal to the specified value.
Definition at line 5042 of file X86ISelLowering.cpp.
References llvm::SM_SentinelUndef.
Referenced by isSequentialOrUndefInRange(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsLanePermuteAndPermute(), and matchUnaryVectorShuffle().
◆ isUndefOrInRange() [1/2]
|
static |
Return true if Val is undef or if its value falls within the specified range (L, H].
Definition at line 5076 of file X86ISelLowering.cpp.
References isInRange(), and llvm::SM_SentinelUndef.
Referenced by getFauxShuffleMask(), getUnderlyingExtractedFromVec(), isUndefOrInRange(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsUNPCKAndPermute(), matchBinaryPermuteVectorShuffle(), and matchUnaryPermuteVectorShuffle().
◆ isUndefOrInRange() [2/2]
Return true if every element in Mask is undef or if its value falls within the specified range (L, H].
Definition at line 5082 of file X86ISelLowering.cpp.
References isUndefOrInRange().
◆ isUndefOrZero()
|
static |
Val is either the undef or zero sentinel value.
Definition at line 5047 of file X86ISelLowering.cpp.
References llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineTargetShuffle(), combineX86ShuffleChain(), isRepeatedTargetShuffleMask(), isSequentialOrUndefOrZeroInRange(), isUndefOrZeroInRange(), isUndefOrZeroOrInRange(), matchUnaryVectorShuffle(), and matchVectorShuffleWithUNPCK().
◆ isUndefOrZeroInRange()
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.
Definition at line 5129 of file X86ISelLowering.cpp.
References isUndefOrZero().
Referenced by matchBinaryPermuteVectorShuffle(), matchUnaryVectorShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
◆ isUndefOrZeroOrInRange() [1/2]
|
static |
Return true if Val is undef, zero or if its value falls within the specified range (L, H].
Definition at line 5092 of file X86ISelLowering.cpp.
References isInRange(), and isUndefOrZero().
Referenced by combineX86ShuffleChain(), and isUndefOrZeroOrInRange().
◆ isUndefOrZeroOrInRange() [2/2]
Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].
Definition at line 5098 of file X86ISelLowering.cpp.
References isUndefOrZeroOrInRange().
◆ isUnpackWdShuffleMask()
Definition at line 9675 of file X86ISelLowering.cpp.
References llvm::createUnpackShuffleMask(), isTargetShuffleEquivalent(), llvm::MVT::v8f32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
◆ isUseOfShuffle()
Definition at line 7376 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, isTargetShuffle(), and llvm::SDNode::uses().
Referenced by lowerBuildVectorAsBroadcast().
◆ isX86CCUnsigned()
Return true if the condition is an unsigned comparison operation.
Definition at line 4649 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, and llvm_unreachable.
Referenced by EmitTest().
◆ isX86LogicalCmp()
Return true if opcode is a X86 logical comparison.
Definition at line 19883 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::X86ISD::COMI, llvm::SDValue::getOpcode(), llvm::SDValue::getResNo(), llvm::X86ISD::OR, llvm::X86ISD::SAHF, llvm::X86ISD::SBB, llvm::X86ISD::SMUL, llvm::X86ISD::SUB, llvm::X86ISD::UCOMI, llvm::X86ISD::UMUL, and llvm::X86ISD::XOR.
Referenced by isTruncWithZeroHighBitsInput(), and isXor1OfSetCC().
◆ isXor1OfSetCC()
Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the SETCC node has a single use.
Definition at line 20743 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, llvm::ISD::AND, assert(), llvm::ISD::BR, llvm::X86ISD::BRCOND, llvm::X86ISD::BT, llvm::X86ISD::CMP, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_O, llvm::X86::COND_P, llvm::StringRef::empty(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::SDValue::getResNo(), llvm::X86TargetLowering::getStackProbeSymbolName(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), getX86XALUOOp(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::MVT::i32, llvm::MVT::i8, isAndOrOfSetCCs(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::X86Subtarget::isOSWindows(), llvm::X86Subtarget::isTargetMachO(), isTruncWithZeroHighBitsInput(), isX86LogicalCmp(), llvm::Lower, LowerAndToBT(), LowerXALUO(), llvm::ISD::OR, llvm::ISD::SADDO, llvm::X86ISD::SETCC, llvm::ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETUNE, llvm::MachineFunction::shouldSplitStack(), llvm::X86ISD::SMUL, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::X86ISD::SUB, llvm::ISD::UADDO, llvm::X86ISD::UMUL, llvm::ISD::UMULO, llvm::SelectionDAG::UpdateNodeOperands(), llvm::SDNode::use_begin(), llvm::ISD::USUBO, and llvm::ISD::XOR.
◆ lower128BitVectorShuffle()
|
static |
Dispatching routine to lower various 128-bit x86 vector shuffles.
This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.
Definition at line 13664 of file X86ISelLowering.cpp.
References llvm_unreachable, lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), llvm::MVT::SimpleTy, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::MVT::v8i16.
Referenced by lowerVectorShuffle().
◆ lower1BitVectorShuffle()
|
static |
Definition at line 15856 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::APInt::countLeadingOnes(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm_unreachable, match1BitShuffleAsKSHIFT(), llvm::PowerOf2Floor(), llvm::ISD::SETGT, llvm::ISD::SIGN_EXTEND, llvm::MVT::SimpleTy, llvm::ArrayRef< T >::size(), llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v2i1, llvm::MVT::v2i64, llvm::MVT::v32i1, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v4i1, llvm::MVT::v4i32, llvm::MVT::v64i1, llvm::MVT::v64i8, llvm::MVT::v8i1, llvm::MVT::v8i32, and llvm::MVT::v8i64.
Referenced by lowerVectorShuffle().
◆ lower256BitVectorShuffle()
|
static |
High-level routine to lower various 256-bit x86 vector shuffles.
This routine either breaks down the specific type of a 256-bit x86 vector shuffle or splits it into two 128-bit shuffles and fuses the results back together based on the available instructions.
Definition at line 15232 of file X86ISelLowering.cpp.
References llvm::count_if(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::isInteger(), llvm_unreachable, lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleWithUndefHalf(), llvm::MVT::SimpleTy, splitAndLowerVectorShuffle(), llvm::MVT::v16i16, llvm::MVT::v32i8, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v8f32, and llvm::MVT::v8i32.
Referenced by lowerVectorShuffle().
◆ Lower256IntUnary()
|
static |
Definition at line 23252 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), and LowerVectorIntUnary().
Referenced by LowerABS(), LowerBITREVERSE(), LowerBITREVERSE_XOP(), LowerCTTZ(), LowerVectorCTLZ(), and LowerVectorCTPOP().
◆ Lower256IntVSETCC()
|
static |
Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back.
Definition at line 19245 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), and llvm::ISD::SETCC.
Referenced by LowerVSETCC().
◆ lower512BitVectorShuffle()
|
static |
High-level routine to lower various 512-bit x86 vector shuffles.
This routine either breaks down the specific type of a 512-bit x86 vector shuffle or splits it into two 256-bit shuffles and fuses the results back together based on the available instructions.
Definition at line 15768 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), llvm::X86Subtarget::hasAVX512(), llvm_unreachable, lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleWithUndefHalf(), llvm::MVT::SimpleTy, llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, llvm::MVT::v16i32, llvm::MVT::v32i16, llvm::MVT::v64i8, llvm::MVT::v8f64, and llvm::MVT::v8i64.
Referenced by lowerVectorShuffle().
◆ Lower512IntUnary()
|
static |
Definition at line 23260 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::MVT::is512BitVector(), llvm::MVT::isInteger(), and LowerVectorIntUnary().
Referenced by LowerVectorCTLZ(), and LowerVectorCTPOP().
◆ LowerABS()
|
static |
Definition at line 23573 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BLENDV, llvm::X86ISD::CMOV, llvm::X86::COND_GE, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), Lower256IntUnary(), llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::MVT::v2i64, and llvm::MVT::v4i64.
Referenced by llvm::PPCTargetLowering::functionArgumentNeedsConsecutiveRegisters(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerADDSAT_SUBSAT()
|
static |
Definition at line 23551 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::i1, llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), llvm_unreachable, llvm::ISD::OR, llvm::ISD::SADDSAT, split256IntArith(), llvm::ISD::SSUBSAT, llvm::ISD::UADDSAT, and llvm::ISD::USUBSAT.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ lowerAddSub()
|
static |
Definition at line 23535 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), lowerAddSubToHorizontalOp(), split256IntArith(), and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerADDSUBCARRY()
|
static |
Definition at line 25770 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::ISD::ADDCARRY, llvm::X86::COND_B, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::MVT::i1, llvm::MVT::i32, llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::MERGE_VALUES, llvm::X86ISD::SBB, and llvm::ISD::TRUNCATE.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ lowerAddSubToHorizontalOp()
|
static |
Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation.
Definition at line 18510 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::X86ISD::HADD, llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), HOp, llvm::X86ISD::HSUB, llvm::MVT::isFloatingPoint(), llvm_unreachable, shouldUseHorizontalOp(), llvm::ISD::SUB, std::swap(), and X.
Referenced by lowerAddSub(), and lowerFaddFsub().
◆ LowerADJUST_TRAMPOLINE()
|
static |
Definition at line 23011 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::AMDGPU::HSAMD::Kernel::Key::Attrs, llvm::CallingConv::C, llvm::MachineFrameInfo::CreateStackObject(), E, llvm::N86::EAX, llvm::N86::ECX, llvm::CallingConv::Fast, llvm::X86ISD::FNSTCW16m, llvm::Function::getAttributes(), llvm::Function::getCallingConv(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::MCRegisterInfo::getEncodingValue(), llvm::SelectionDAG::getEntryNode(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::X86Subtarget::getFrameLowering(), llvm::Function::getFunctionType(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::X86Subtarget::getRegisterInfo(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::TargetFrameLowering::getStackAlignment(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getVTList(), llvm::AttributeList::hasAttribute(), I, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::Attribute::InReg, llvm::X86Subtarget::is64Bit(), llvm::AttributeList::isEmpty(), llvm::Function::isVarArg(), llvm_unreachable, llvm::MachineMemOperand::MOStore, llvm::ISD::OR, llvm::MVT::Other, llvm::FunctionType::param_begin(), llvm::FunctionType::param_end(), llvm::report_fatal_error(), llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::TokenFactor, TRI, llvm::ISD::TRUNCATE, llvm::CallingConv::X86_FastCall, llvm::CallingConv::X86_StdCall, llvm::CallingConv::X86_ThisCall, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::PPCTargetLowering::functionArgumentNeedsConsecutiveRegisters(), llvm::XCoreTargetLowering::getExceptionSelectorRegister(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerAndToBT()
|
static |
Result of 'and' is compared against zero.
Change to a BT node if possible. Returns the BT node and the condition code needed to use it.
Definition at line 19119 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::X86ISD::BT, llvm::SelectionDAG::computeKnownBits(), llvm::X86::COND_AE, llvm::X86::COND_B, llvm::ISD::Constant, llvm::KnownBits::countMinLeadingZeros(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isOneConstant(), llvm::isPowerOf2_64(), llvm::isUInt< 32 >(), llvm::isUInt< 8 >(), llvm::Log2_64_Ceil(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::Function::optForSize(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::ISD::SRL, std::swap(), and llvm::ISD::TRUNCATE.
Referenced by EmitKORTEST(), isTruncWithZeroHighBitsInput(), and isXor1OfSetCC().
◆ LowerANY_EXTEND()
|
static |
Definition at line 20277 of file X86ISelLowering.cpp.
References assert(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), llvm::MVT::i1, llvm::tgtok::In, LowerAVXExtend(), and LowerSIGN_EXTEND_Mask().
Referenced by llvm::HexagonTargetLowering::isCtlzFast(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerAsSplatVectorLoad()
|
static |
Definition at line 7118 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::MVT::f32, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::i32, llvm::SelectionDAG::InferPtrAlignment(), llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::ISD::isNormalLoad(), llvm::ARM_MB::LD, llvm::BitmaskEnumDetail::Mask(), and llvm::MachineFrameInfo::setObjectAlignment().
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ LowerATOMIC_FENCE()
|
static |
Definition at line 25232 of file X86ISelLowering.cpp.
References llvm::N86::ESP, llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasMFence(), llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::MEMBARRIER, llvm::X86ISD::MFENCE, llvm::MVT::Other, llvm::SequentiallyConsistent, and llvm::SyncScope::System.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerATOMIC_STORE()
|
static |
Definition at line 25746 of file X86ISelLowering.cpp.
Referenced by llvm::XCoreTargetLowering::getExceptionSelectorRegister(), and llvm::X86TargetLowering::LowerOperation().
◆ lowerAtomicArith()
|
static |
Lower atomic_load_ops into LOCK-prefixed operations.
Definition at line 25713 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_SUB, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getConstant(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SDValue::getValue(), llvm::SDNode::hasAnyUseOfValue(), lowerAtomicArithWithLOCK(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::ISD::SUB.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ lowerAtomicArithWithLOCK()
|
static |
Definition at line 25681 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_LOAD_XOR, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::X86ISD::LADD, llvm::X86ISD::LAND, llvm_unreachable, llvm::X86ISD::LOR, llvm::X86ISD::LSUB, llvm::X86ISD::LXOR, N, and llvm::MVT::Other.
Referenced by combineSetCCAtomicArith(), and lowerAtomicArith().
◆ LowerAVXCONCAT_VECTORS()
|
static |
Definition at line 9236 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MipsISD::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm::MipsISD::Lo, and llvm::SDNode::ops().
Referenced by LowerCONCAT_VECTORS().
◆ LowerAVXExtend()
|
static |
Definition at line 17963 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, ExperimentalVectorWideningLegalization, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::tgtok::In, llvm::MVT::isVector(), llvm::RegState::Undef, llvm::MVT::v16i8, llvm::MVT::v8i64, llvm::MVT::v8i8, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by LowerANY_EXTEND(), and LowerZERO_EXTEND().
◆ LowerBITCAST()
|
static |
Definition at line 25324 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_ELEMENT, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), getPMOVMSKB(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasMMX(), llvm::X86Subtarget::hasSSE2(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is64Bit(), llvm::MVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::MipsISD::Lo, llvm::X86ISD::MOVDQ2Q, llvm::ISD::SCALAR_TO_VECTOR, llvm::SelectionDAG::SplitVector(), llvm::MVT::v16i1, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v32i1, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v4i16, llvm::MVT::v64i1, llvm::MVT::v64i8, llvm::MVT::v8i8, and llvm::MVT::x86mmx.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerBITREVERSE()
|
static |
Definition at line 25630 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasXOP(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::tgtok::In, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MipsISD::Lo, Lower256IntUnary(), LowerBITREVERSE_XOP(), llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T >::push_back(), and llvm::ISD::SRL.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerBITREVERSE_XOP()
|
static |
Definition at line 25585 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITREVERSE, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i8, llvm::tgtok::In, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::isVector(), Lower256IntUnary(), llvm::BitmaskEnumDetail::Mask(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v16i8, and llvm::X86ISD::VPPERM.
Referenced by LowerBITREVERSE().
◆ LowerBUILD_VECTORAsVariablePermute()
|
static |
Definition at line 8808 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), assert(), buildFromShuffleMostly(), C, concatSubVectors(), llvm::countTrailingZeros(), createVariablePermute(), llvm::dyn_cast(), E, EltsFromConsecutiveLoads(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::MVT::f64, llvm::ConstantInt::get(), llvm::ConstantFP::get(), llvm::ConstantVector::get(), llvm::UndefValue::get(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), getMOVL(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), getShuffleVectorZeroOrUndef(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), getUnpackl(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::TargetLoweringBase::getVectorIdxTy(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getVShift(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::SmallSet< T, N, C >::insert(), insert128BitVector(), llvm::ISD::INSERT_VECTOR_ELT, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::X86Subtarget::is64Bit(), llvm::MVT::isFloatingPoint(), llvm::SDNode::isOnlyUserOf(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::SDValue::isUndef(), isZero(), llvm::X86::isZeroNode(), llvm::ARM_MB::LD, llvm_unreachable, llvm::Lower, LowerAsSplatVectorLoad(), LowerBUILD_VECTORvXi1(), lowerBuildVectorAsBroadcast(), lowerBuildVectorToBitOp(), LowerBuildVectorv16i8(), LowerBuildVectorv4x32(), LowerBuildVectorv8i16(), lowerToAddSubOrFMAddSub(), LowerToHorizontalOp(), llvm::BitmaskEnumDetail::Mask(), materializeVectorConstant(), llvm::SDNode::op_begin(), llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase::size(), llvm::SmallSet< T, N, C >::size(), llvm::SM_SentinelUndef, llvm::Upper, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v4i32, llvm::X86ISD::VBROADCAST, llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::ZERO_EXTEND.
◆ LowerBUILD_VECTORvXi1()
|
static |
Definition at line 7771 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, ConvertI1VectorToInteger(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, llvm::tgtok::In, llvm::ISD::INSERT_VECTOR_ELT, llvm::X86Subtarget::is64Bit(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), llvm::Lower, llvm::max(), llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorBase::size(), llvm::Upper, llvm::MVT::v32i1, llvm::MVT::v64i1, and llvm::MVT::v8i1.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ lowerBuildVectorAsBroadcast()
|
static |
Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a.
A single scalar load, or a constant. b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
The VBROADCAST node is returned when a pattern is found, or SDValue() otherwise.
Definition at line 7430 of file X86ISelLowering.cpp.
References llvm::lltok::APFloat, assert(), llvm::ISD::BITCAST, C, llvm::ISD::Constant, llvm::ISD::ConstantFP, llvm::BitVector::count(), llvm::HexagonISD::CP, llvm::ConstantFP::get(), getAlignment(), llvm::SelectionDAG::getBitcast(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), getConstantVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getFloatingPointVT(), llvm::MachineFunction::getFunction(), llvm::Constant::getIntegerValue(), llvm::MVT::getIntegerVT(), llvm::Type::getIntNTy(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasCDI(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i32, llvm::MVT::i64, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::X86Subtarget::is32Bit(), llvm::MVT::is512BitVector(), llvm::BuildVectorSDNode::isConstantSplat(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::SDNode::isOnlyUserOf(), isSplatZeroExtended(), isUseOfShuffle(), llvm::EVT::isVector(), llvm::Function::optForSize(), llvm::X86ISD::SUBV_BROADCAST, llvm::RegState::Undef, llvm::MVT::v16i1, llvm::MVT::v8i1, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCASTM, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ LowerBuildVectorAsInsert()
|
static |
Definition at line 6862 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i32, llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v16i8, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::X86ISD::VZEXT_MOVL.
Referenced by LowerBuildVectorv16i8(), and LowerBuildVectorv8i16().
◆ lowerBuildVectorToBitOp()
|
static |
If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.
NOTE: Its not in our interest to start make a general purpose vectorizer from this, but enough scalar bit operations are created from the later legalization + scalarization stages to need basic support.
Definition at line 8469 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm::SDNode::ops(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T >::push_back(), and llvm::ISD::XOR.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ LowerBuildVectorv16i8()
|
static |
Custom lower build_vector of v16i8.
Definition at line 6905 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, LowerBuildVectorAsInsert(), llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SHL, llvm::MVT::v16i8, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VZEXT_MOVL, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ LowerBuildVectorv4x32()
|
static |
Custom lower build_vector of v4i32 or v4f32.
Definition at line 6982 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasXOP(), llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::X86ISD::MOVDDUP, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, and llvm::MVT::v4f32.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ LowerBuildVectorv8i16()
|
static |
Custom lower build_vector of v8i16.
Definition at line 6969 of file X86ISelLowering.cpp.
References llvm::X86Subtarget::hasSSE41(), and LowerBuildVectorAsInsert().
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ LowerCMP_SWAP()
|
static |
Definition at line 25266 of file X86ISelLowering.cpp.
References llvm::AArch64CC::AL, assert(), llvm::X86::COND_E, llvm::N86::EAX, llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getOperand(), getSETCC(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::X86ISD::LCMPXCHG_DAG, llvm_unreachable, llvm::MVT::Other, Reg, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::MVT::SimpleTy, llvm::size(), Success, and T.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerCONCAT_VECTORS()
|
static |
Definition at line 9417 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNumOperands(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerAVXCONCAT_VECTORS(), and LowerCONCAT_VECTORSvXi1().
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerCONCAT_VECTORSvXi1()
|
static |
Definition at line 9346 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::countTrailingZeros(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MipsISD::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::isPowerOf2_32(), isTypePromotionOfi1ZeroUpBits(), llvm::SDValue::isUndef(), llvm::MipsISD::Lo, llvm::SDNode::ops(), and widenSubVector().
Referenced by LowerCONCAT_VECTORS().
◆ LowerCTLZ()
|
static |
Definition at line 23407 of file X86ISelLowering.cpp.
References llvm::X86ISD::BSR, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::isVector(), LowerVectorCTLZ(), llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerCTPOP()
|
static |
Definition at line 25578 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::MVT::isVector(), and LowerVectorCTPOP().
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerCTTZ()
|
static |
Definition at line 23449 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BSF, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::ISD::CTTZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::MVT::isVector(), and Lower256IntUnary().
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerEXTEND_VECTOR_INREG()
|
static |
Definition at line 20293 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::MipsISD::Hi, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::tgtok::In, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MipsISD::Lo, llvm::BitmaskEnumDetail::Mask(), llvm::max(), llvm::ISD::SETGT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::SM_SentinelUndef, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::X86ISD::VSRAI, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerEXTRACT_SUBVECTOR()
|
static |
Definition at line 16696 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::X86Subtarget::classifyBlockAddressReference(), llvm::X86Subtarget::classifyGlobalReference(), llvm::X86Subtarget::classifyLocalReference(), llvm::HexagonISD::CP, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ConstantPoolSDNode::getAlignment(), llvm::TargetMachine::getCodeModel(), llvm::SelectionDAG::getConstant(), llvm::ConstantPoolSDNode::getConstVal(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFunction(), llvm::MachinePointerInfo::getGOT(), llvm::JumpTableSDNode::getIndex(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::getOffset(), llvm::ConstantPoolSDNode::getOffset(), llvm::SDValue::getOperand(), llvm::GlobalValue::getParent(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getSimpleValueType(), llvm::object::getSymbol(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetBlockAddress(), llvm::SelectionDAG::getTargetConstantPool(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SelectionDAG::getTargetJumpTable(), llvm::TargetLoweringBase::getTargetMachine(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::GlobalBaseReg, llvm::X86Subtarget::hasDQI(), llvm::MVT::i1, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::GlobalValue::isAbsoluteSymbolRef(), llvm::isGlobalRelativeToPICBase(), llvm::isGlobalStubReference(), llvm::X86::isOffsetSuitableForCodeModel(), llvm::X86Subtarget::isPICStyleRIPRel(), llvm::HexagonISD::JT, llvm::CodeModel::Kernel, llvm::X86ISD::KSHIFTR, llvm::X86II::MO_GOTPCREL, llvm::X86II::MO_NO_FLAG, llvm::Mod, llvm::CodeModel::Small, llvm::MVT::v16i1, llvm::MVT::v8i1, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.
Referenced by llvm::HexagonTargetLowering::isCtlzFast(), llvm::X86TargetLowering::LowerOperation(), and llvm::AArch64TargetLowering::supportSwiftError().
◆ LowerEXTRACT_VECTOR_ELT_SSE4()
|
static |
Definition at line 16262 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::is128BitVector(), llvm::isNullConstant(), llvm::X86ISD::PEXTRB, llvm::ISD::STORE, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), and llvm::MVT::v4i32.
Referenced by ExtractBitFromMaskVector().
◆ LowerFABSorFNEG()
|
static |
The only differences between FABS and FNEG are the mask and the logic op.
FNEG also has a folding opportunity for FNEG(FABS(x)).
Definition at line 18592 of file X86ISelLowering.cpp.
References llvm::lltok::APFloat, assert(), llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f64, llvm::ISD::FABS, llvm::X86ISD::FAND, llvm::ISD::FNEG, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SDNode::uses(), llvm::MVT::v16f32, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, and llvm::MVT::v8f64.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ lowerFaddFsub()
|
static |
Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation.
Definition at line 18583 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::f32, llvm::MVT::f64, llvm::SDValue::getValueType(), and lowerAddSubToHorizontalOp().
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerFCOPYSIGN()
|
static |
Definition at line 18652 of file X86ISelLowering.cpp.
References llvm::lltok::APFloat, assert(), llvm::MVT::bitsGT(), llvm::MVT::bitsLT(), llvm::APFloat::clearSign(), llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::isConstOrConstSplatFP(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v16f32, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, and llvm::MVT::v8f64.
Referenced by llvm::ARMTargetLowering::hasStandaloneRem(), llvm::X86TargetLowering::LowerOperation(), and llvm::AArch64TargetLowering::supportSwiftError().
◆ LowerFGETSIGN()
|
static |
Definition at line 18719 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::i32, llvm::X86ISD::MOVMSK, llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v2f64, and llvm::MVT::v4f32.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerFP_EXTEND()
|
static |
Definition at line 18485 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::tgtok::In, llvm::MVT::v2f32, llvm::MVT::v4f32, and llvm::X86ISD::VFPEXT.
Referenced by llvm::ARMTargetLowering::hasStandaloneRem(), llvm::X86TargetLowering::LowerOperation(), and llvm::AArch64TargetLowering::supportSwiftError().
◆ LowerFSINCOS()
|
static |
Definition at line 25799 of file X86ISelLowering.cpp.
References Arg, llvm::AMDGPU::HSAMD::Kernel::Key::Args, assert(), llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::StructType::get(), llvm::VectorType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExternalSymbol(), llvm::SelectionDAG::getIntPtrConstant(), llvm::TargetLoweringBase::getLibcallName(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getTypeForEVT(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isTargetDarwin(), llvm::TargetLowering::LowerCallTo(), llvm::ISD::MERGE_VALUES, llvm::TargetLowering::CallLoweringInfo::setChain(), and llvm::TargetLowering::CallLoweringInfo::setDebugLoc().
Referenced by llvm::ARMTargetLowering::hasStandaloneRem(), llvm::X86TargetLowering::LowerOperation(), and llvm::AArch64TargetLowering::supportSwiftError().
◆ LowerFunnelShift()
|
static |
Definition at line 17245 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasVBMI2(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, isConstantSplat(), llvm::X86Subtarget::isSHLDSlow(), llvm::MVT::isVector(), llvm::Function::optForSize(), llvm::X86ISD::SHLD, llvm::X86ISD::SHRD, std::swap(), llvm::X86ISD::VSHLD, llvm::X86ISD::VSHLDV, llvm::X86ISD::VSHRD, and llvm::X86ISD::VSHRDV.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerHorizontalByteSum()
|
static |
Compute the horizontal sum of bytes in V for the elements of VT.
Requires V to be a byte vector and VT to be an integer vector type with wider elements than V's type. The width of the elements of VT determines how many bytes of V are summed horizontally to produce each element of the result.
Definition at line 25423 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), High, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, llvm::ISD::SHL, and llvm::ISD::SRL.
Referenced by LowerVectorCTPOP().
◆ LowerI64IntToFP_AVX512DQ()
|
static |
Definition at line 17297 of file X86ISelLowering.cpp.
References assert(), llvm::X86TargetLowering::BuildFILD(), llvm::ISD::CONCAT_VECTORS, llvm::MachineFrameInfo::CreateStackObject(), llvm::X86ISD::CVTSI2P, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::MachineFunction::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is64Bit(), llvm::X86TargetLowering::isScalarFPTypeInSSEReg(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SINT_TO_FP, Size, llvm::ISD::UINT_TO_FP, llvm::MVT::v2f64, llvm::MVT::v2i32, and llvm::MVT::v4i32.
Referenced by lowerUINT_TO_FP_vec().
◆ LowerINSERT_SUBVECTOR()
|
static |
Definition at line 16689 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, and insert1BitVector().
Referenced by llvm::HexagonTargetLowering::isCtlzFast(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerINTRINSIC_W_CHAIN()
|
static |
Definition at line 22622 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::X86ISD::CMOV, llvm::X86::COND_B, llvm::X86::COND_NE, llvm::MachineFrameInfo::CreateFixedObject(), llvm::Depth, llvm::dyn_cast(), EmitMaskedTruncSStore(), EmitTruncSStore(), llvm::GATHER, llvm::GATHER_AVX2, getAVX2GatherNode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), getExtendedControlRegister(), llvm::X86MachineFunctionInfo::getFAIndex(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), getGatherNode(), llvm::MachineFunction::getInfo(), llvm::getIntrinsicWithChain(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMaskedStore(), getMaskNode(), llvm::TargetMachine::getMCAsmInfo(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), getPrefetchNode(), getReadPerformanceCounter(), getReadTimeStampCounter(), llvm::X86Subtarget::getRegisterInfo(), llvm::X86TargetLowering::getReturnAddressFrameIndex(), getScatterNode(), getSETCC(), llvm::X86RegisterInfo::getSlotSize(), llvm::MachineFunction::getTarget(), llvm::SelectionDAG::getTruncStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isAllOnesConstant(), llvm_unreachable, llvm::X86ISD::LWPINS, MarkEHGuard(), MarkEHRegistrationNode(), llvm::ISD::MERGE_VALUES, llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, Operation, llvm::MVT::Other, llvm::PREFETCH, llvm::RDPMC, llvm::RDRAND, llvm::RDSEED, llvm::RDTSC, Results, llvm::MipsISD::Ret, llvm::SCATTER, llvm::X86MachineFunctionInfo::setFAIndex(), llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::MachineFrameInfo::setHasCopyImplyingStackAdjustment(), llvm::MachineFrameInfo::setReturnAddressIsTaken(), llvm::X86ISD::TPAUSE, llvm::TRUNCATE_TO_MEM_VI16, llvm::TRUNCATE_TO_MEM_VI32, llvm::TRUNCATE_TO_MEM_VI8, llvm::IntrinsicData::Type, llvm::X86ISD::UMWAIT, llvm::MCAsmInfo::usesWindowsCFI(), llvm::TargetLowering::verifyReturnAddressArgumentIsConstant(), llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, llvm::X86ISD::VTRUNCUS, llvm::Intrinsic::x86_flags_read_u32, llvm::Intrinsic::x86_flags_read_u64, llvm::Intrinsic::x86_flags_write_u32, llvm::Intrinsic::x86_flags_write_u64, llvm::Intrinsic::x86_lwpins32, llvm::Intrinsic::x86_lwpins64, llvm::Intrinsic::x86_seh_ehguard, llvm::Intrinsic::x86_seh_ehregnode, llvm::Intrinsic::x86_tpause, llvm::Intrinsic::x86_umwait, llvm::XGETBV, llvm::XTEST, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerIntVSETCC_AVX512()
|
static |
Definition at line 19273 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::SETEQ, llvm::ISD::SETLT, llvm::ISD::SETNE, and std::swap().
Referenced by LowerVSETCC().
◆ LowerLoad()
|
static |
Definition at line 20525 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, assert(), llvm::MipsISD::Ext, llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), getExtendInVec(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, llvm::MVT::integer_valuetypes(), llvm::MVT::isInteger(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::SPII::Load, llvm::MinAlign(), llvm::MVT::Other, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SEXTLOAD, llvm::ISD::TokenFactor, llvm::MVT::v16i1, llvm::MVT::v8i64, and llvm::MVT::v8i8.
Referenced by llvm::HexagonTargetLowering::isCtlzFast(), and llvm::X86TargetLowering::LowerOperation().
◆ lowerMasksToReg()
|
static |
Lowers masks values (v*i1) to the local register values.
- Returns
- DAG node after lowering to register type
Definition at line 2367 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::v16i1, llvm::MVT::v1i1, llvm::MVT::v32i1, llvm::MVT::v64i1, and llvm::MVT::v8i1.
Referenced by getMOVL(), and Passv64i1ArgInRegs().
◆ LowerMGATHER()
|
static |
Definition at line 26086 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDNode::getGluedNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetMemSDNode(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::is512BitVector(), llvm::BitmaskEnumDetail::Mask(), llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), and llvm::MVT::v2i32.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerMINMAX()
|
static |
Definition at line 23607 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignedMinValue(), llvm::SDValue::getSimpleValueType(), llvm::MVT::i64, llvm::MVT::is256BitVector(), llvm_unreachable, llvm::ISD::SETGT, llvm::ISD::SETLT, llvm::ISD::SETUGT, llvm::ISD::SETULT, llvm::ISD::SMAX, llvm::ISD::SMIN, split256IntArith(), llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::MVT::v8i16, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerMLOAD()
|
static |
Definition at line 25996 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::MaskedLoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MaskedLoadSDNode::getPassThru(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::MaskedLoadSDNode::isExpandingLoad(), and llvm::BitmaskEnumDetail::Mask().
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerMSCATTER()
|
static |
Definition at line 25904 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, ExtendToType(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMaskedScatter(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetMemSDNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::is512BitVector(), llvm::BitmaskEnumDetail::Mask(), llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::MVT::v2f32, llvm::MVT::v2i1, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i1, and llvm::MVT::v4i32.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerMSTORE()
|
static |
Definition at line 26045 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MaskedStoreSDNode::getValue(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::MVT::is512BitVector(), llvm::MaskedStoreSDNode::isCompressingStore(), llvm::MaskedStoreSDNode::isTruncatingStore(), and llvm::BitmaskEnumDetail::Mask().
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerMUL()
|
static |
Definition at line 23646 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), B, llvm::X86Subtarget::canExtendTo512BW(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), getTargetVShiftByConstNode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::APInt::isSubsetOf(), llvm::ISD::MUL, llvm::X86ISD::PACKUS, llvm::X86ISD::PMULUDQ, llvm::SmallVectorTemplateBase< T >::push_back(), split256IntArith(), llvm::ISD::TRUNCATE, llvm::RegState::Undef, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v64i8, llvm::MVT::v8i64, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and llvm::KnownBits::Zero.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerMULH()
|
static |
Definition at line 23791 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::AMDGPU::HSAMD::Kernel::Key::Args, assert(), B, llvm::X86Subtarget::canExtendTo512BW(), llvm::SelectionDAG::CreateStackTemporary(), extract128BitVector(), Fixup, llvm::PointerType::get(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExternalSymbol(), llvm::SelectionDAG::getIntPtrConstant(), llvm::TargetLoweringBase::getLibcallCallingConv(), llvm::TargetLoweringBase::getLibcallName(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), getTargetVShiftByConstNode(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i16, llvm::MVT::i64, llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isInteger(), llvm::X86Subtarget::isTargetWin64(), llvm_unreachable, llvm::MipsISD::Lo, llvm::TargetLowering::LowerCallTo(), llvm::makeArrayRef(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::X86ISD::PACKUS, llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SDIV, llvm::ISD::SDIVREM, llvm::TargetLowering::CallLoweringInfo::setChain(), llvm::TargetLowering::CallLoweringInfo::setDebugLoc(), llvm::ISD::SETGT, llvm::TargetLowering::CallLoweringInfo::setLibCallee(), llvm::TargetLowering::CallLoweringInfo::setSExtResult(), llvm::TargetLowering::CallLoweringInfo::setZExtResult(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, split256IntArith(), split512IntArith(), llvm::ISD::SREM, llvm::ISD::SUB, T1, llvm::ISD::TRUNCATE, llvm::ISD::UDIV, llvm::ISD::UDIVREM, llvm::ISD::UREM, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i32, llvm::MVT::v64i8, llvm::MVT::v8i32, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerREADCYCLECOUNTER()
|
static |
Definition at line 22550 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), getReadTimeStampCounter(), llvm::X86ISD::RDTSC_DAG, and Results.
Referenced by llvm::HexagonTargetLowering::isCtlzFast(), and llvm::X86TargetLowering::LowerOperation().
◆ lowerRegToMasks()
|
static |
The function will lower a register of various sizes (8/16/32/64) to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1)
- Returns
- a DAG node contains the operand after lowering to mask type.
Definition at line 2744 of file X86ISelLowering.cpp.
References llvm::CCState::AnalyzeCallResult(), assert(), llvm::CCValAssign::convertToReg(), E, errorUnsupported(), llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::f80, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getIntPtrConstant(), llvm::CCValAssign::getLocReg(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::X86Subtarget::getRegisterInfo(), llvm::MVT::getScalarType(), llvm::EVT::getSimpleVT(), getv64i1Argument(), llvm::SDValue::getValue(), llvm::CCValAssign::getValVT(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasX87(), I, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MipsISD::Ins, llvm::X86Subtarget::is64Bit(), llvm::CCValAssign::isExtInLoc(), llvm::X86TargetLowering::isScalarFPTypeInSSEReg(), llvm::MCRegisterInfo::DiffListIterator::isValid(), llvm::MVT::isVector(), llvm_unreachable, llvm::CCValAssign::needsCustom(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::report_fatal_error(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::SimpleTy, llvm::SmallVectorBase::size(), TRI, llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v1i1, llvm::MVT::v32i1, llvm::MVT::v64i1, and llvm::MVT::v8i1.
Referenced by isSortedByValueNo().
◆ LowerRotate()
|
static |
Definition at line 24916 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::AtomicRMWInst::Add, llvm::ISD::AND, llvm::AtomicRMWInst::And, assert(), C, llvm::TargetLoweringBase::CmpXChg, convertShiftLeftToScale(), llvm::Instruction::eraseFromParent(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::Intrinsic::getDeclaration(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOrdering(), llvm::AtomicRMWInst::getPointerOperand(), llvm::LoadInst::getPointerOperandType(), llvm::Type::getPrimitiveSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(), llvm::AtomicRMWInst::getSyncScopeID(), getTargetConstantBitsFromNode(), llvm::Value::getType(), llvm::StoreInst::getValueOperand(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasCmpxchg16b(), llvm::X86Subtarget::hasMFence(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasXOP(), llvm::MipsISD::Hi, llvm::MVT::i16, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::X86Subtarget::is64Bit(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isSplatValue(), llvm::MVT::isVector(), llvm_unreachable, llvm::MipsISD::Lo, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::ISD::MUL, llvm::ISD::MULHU, llvm::AtomicRMWInst::Nand, llvm::TargetLoweringBase::None, llvm::ISD::OR, llvm::AtomicRMWInst::Or, llvm::X86ISD::PCMPGT, llvm::X86ISD::PMULUDQ, llvm::Value::replaceAllUsesWith(), llvm::ISD::ROTL, llvm::LoadInst::setAtomic(), llvm::ISD::SHL, SI, llvm::SyncScope::SingleThread, split256IntArith(), llvm::ISD::SRL, llvm::ISD::SUB, llvm::AtomicRMWInst::Sub, SupportedVectorVarShift(), llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::Value::use_empty(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i32, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::Intrinsic::x86_sse2_mfence, llvm::AtomicRMWInst::Xchg, llvm::AtomicRMWInst::Xor, and llvm::Z.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerSCALAR_TO_VECTOR()
|
static |
Definition at line 16652 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i32, insert128BitVector(), llvm::MVT::is128BitVector(), llvm::X86::isZeroNode(), llvm::ISD::SCALAR_TO_VECTOR, and llvm::MVT::v4i32.
Referenced by llvm::PPCTargetLowering::functionArgumentNeedsConsecutiveRegisters(), llvm::X86TargetLowering::LowerOperation(), and llvm::AArch64TargetLowering::supportSwiftError().
◆ LowerScalarImmediateShift()
|
static |
Definition at line 24117 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::ARCISD::CMP, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE42(), llvm::X86Subtarget::hasXOP(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::is512BitVector(), isConstantSplat(), llvm_unreachable, llvm::Lower, llvm::X86ISD::PCMPGT, llvm::ISD::SETGT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, SupportedVectorShiftWithImm(), llvm::Upper, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i64, llvm::MVT::v64i1, llvm::MVT::v64i8, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::XOR.
Referenced by LowerShift().
◆ LowerScalarVariableShift()
|
static |
Definition at line 24284 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::MVT::bitsGT(), llvm::MVT::bitsLE(), llvm::MVT::bitsLT(), llvm::ISD::BUILD_VECTOR, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), GetSplatValue(), getTargetVShiftByConstNode(), getTargetVShiftNode(), getTargetVShiftUniformOpcode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasXOP(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, SupportedVectorShiftWithBaseAmnt(), llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v64i8, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerShift().
◆ LowerShift()
|
static |
Definition at line 24435 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), C, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), canWidenShuffleElements(), convertShiftLeftToScale(), llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasVLX(), llvm::X86Subtarget::hasXOP(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isKnownNeverZero(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), llvm::MipsISD::Lo, LowerScalarImmediateShift(), LowerScalarVariableShift(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::ISD::OR, llvm::X86ISD::PACKUS, llvm::X86ISD::PCMPGT, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), R2, llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase::size(), llvm::SM_SentinelUndef, split256IntArith(), llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, SupportedVectorVarShift(), llvm::ISD::TRUNCATE, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v64i8, llvm::MVT::v8i16, llvm::MVT::v8i32, llvm::X86ISD::VPSHA, llvm::X86ISD::VPSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, llvm::ISD::XOR, llvm::Z, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ LowerShiftParts()
|
static |
Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.
TODO: Can this be moved to general expansion code?
Definition at line 17198 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::MipsISD::Lo, llvm::ISD::SELECT, llvm::ISD::SETNE, llvm::ISD::SHL, llvm::ISD::SHL_PARTS, llvm::ISD::SRA, llvm::ISD::SRA_PARTS, and llvm::ISD::SRL.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ lowerShuffleAsRepeatedMaskAndLanePermute()
|
static |
Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.
Definition at line 14406 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::max(), and llvm::SM_SentinelUndef.
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
◆ LowerSIGN_EXTEND()
|
static |
Definition at line 20405 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, ExperimentalVectorWideningLegalization, llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::tgtok::In, llvm::MVT::isVector(), LowerSIGN_EXTEND_Mask(), llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::MVT::v16i8, llvm::MVT::v8i64, and llvm::MVT::v8i8.
Referenced by llvm::MSP430TargetLowering::getScalarShiftAmountTy(), llvm::HexagonTargetLowering::isCtlzFast(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerSIGN_EXTEND_Mask()
|
static |
Definition at line 20220 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i32, llvm::tgtok::In, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), SplitAndExtendv16i1(), and llvm::ISD::TRUNCATE.
Referenced by LowerANY_EXTEND(), and LowerSIGN_EXTEND().
◆ LowerStore()
|
static |
Definition at line 20465 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::MemSDNode::getAlignment(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeAction(), llvm::SelectionDAG::getUNDEF(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasDQI(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::X86Subtarget::is64Bit(), llvm::MVT::isInteger(), llvm::StoreSDNode::isTruncatingStore(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::ISD::TRUNCATE, llvm::TargetLoweringBase::TypeWidenVector, and llvm::MVT::v16i1.
Referenced by llvm::HexagonTargetLowering::isCtlzFast(), and llvm::X86TargetLowering::LowerOperation().
◆ lowerToAddSubOrFMAddSub()
|
static |
Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node.
Definition at line 8193 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADDSUB, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUBADD, llvm::SelectionDAG::getNode(), llvm::SDNode::getSimpleValueType(), llvm::MVT::is512BitVector(), isAddSubOrSubAdd(), and isFMAddSubOrFMSubAdd().
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ LowerToHorizontalOp()
|
static |
Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
Definition at line 8337 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), ExpandHorizontalBinOp(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, getHopForBuildVector(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, llvm::MVT::is256BitVector(), isHopBuildVector(), isHorizontalBinOpPart(), llvm::SDValue::isUndef(), llvm::SDNode::isUndef(), llvm::SDNode::op_values(), llvm::ISD::SUB, llvm::MVT::v16i16, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v8f32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ LowerToTLSExecModel()
|
static |
Definition at line 16994 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::AMDGPU::HSAMD::Kernel::Key::Args, llvm::N86::EAX, llvm::TLSModel::GeneralDynamic, llvm::SelectionDAG::getCALLSEQ_END(), llvm::SelectionDAG::getCALLSEQ_START(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExternalSymbol(), llvm::SelectionDAG::getExtLoad(), llvm::MachineFunction::getFrameInfo(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachinePointerInfo::getGOT(), llvm::Type::getInt32PtrTy(), llvm::Type::getInt8PtrTy(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::GlobalAddressSDNode::getOffset(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::GlobalValue::getThreadLocalMode(), llvm::TargetMachine::getTLSModel(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86ISD::GlobalBaseReg, llvm::MVT::Glue, llvm::MVT::i32, llvm::MVT::i8, llvm::TLSModel::InitialExec, llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isPICStyleRIPRel(), llvm::TargetLowering::isPositionIndependent(), llvm::X86Subtarget::isTargetDarwin(), llvm::X86Subtarget::isTargetELF(), llvm::X86Subtarget::isTargetKnownWindowsMSVC(), llvm::X86Subtarget::isTargetWindowsGNU(), llvm::X86Subtarget::isTargetWindowsItanium(), llvm_unreachable, llvm::TLSModel::LocalDynamic, llvm::TLSModel::LocalExec, llvm::GlobalValue::LocalExecTLSModel, llvm::Log2_64_Ceil(), llvm::TargetLowering::LowerToTLSEmulatedModel(), LowerToTLSGeneralDynamicModel32(), LowerToTLSGeneralDynamicModel64(), LowerToTLSLocalDynamicModel(), llvm::X86II::MO_GOTNTPOFF, llvm::X86II::MO_GOTTPOFF, llvm::X86II::MO_INDNTPOFF, llvm::X86II::MO_NTPOFF, llvm::X86II::MO_SECREL, llvm::X86II::MO_TLVP, llvm::X86II::MO_TLVP_PIC_BASE, llvm::X86II::MO_TPOFF, llvm::MVT::Other, Reg, llvm::MachineFrameInfo::setAdjustsStack(), llvm::ISD::SHL, llvm::MipsISD::ThreadPointer, llvm::X86ISD::TLSCALL, llvm::TargetMachine::useEmulatedTLS(), llvm::X86ISD::Wrapper, llvm::X86ISD::WrapperRIP, and llvm::ISD::ZEXTLOAD.
◆ LowerToTLSGeneralDynamicModel32()
|
static |
Definition at line 16934 of file X86ISelLowering.cpp.
References llvm::N86::EAX, llvm::N86::EBX, llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), GetTLSADDR(), llvm::SDValue::getValue(), llvm::X86ISD::GlobalBaseReg, and llvm::X86II::MO_TLSGD.
Referenced by LowerToTLSExecModel().
◆ LowerToTLSGeneralDynamicModel64()
|
static |
Definition at line 16948 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getEntryNode(), GetTLSADDR(), and llvm::X86II::MO_TLSGD.
Referenced by LowerToTLSExecModel().
◆ LowerToTLSLocalDynamicModel()
|
static |
Definition at line 16954 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::N86::EAX, llvm::N86::EBX, llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), GetTLSADDR(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::X86ISD::GlobalBaseReg, llvm::X86MachineFunctionInfo::incNumLocalDynamicTLSAccesses(), llvm::X86II::MO_DTPOFF, llvm::X86II::MO_TLSLD, llvm::X86II::MO_TLSLDM, and llvm::X86ISD::Wrapper.
Referenced by LowerToTLSExecModel().
◆ LowerTruncateVecI1()
|
static |
Definition at line 18214 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::CONCAT_VECTORS, llvm::KnownBits::countMinLeadingZeros(), llvm::X86ISD::CVTTP2SI, llvm::X86ISD::CVTTP2UI, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_UINT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasVLX(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::tgtok::In, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::MipsISD::Lo, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::ISD::SETGT, llvm::ISD::SETNE, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, truncateVectorWithPACK(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2f32, llvm::MVT::v2f64, llvm::MVT::v2i1, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4i1, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8f64, llvm::MVT::v8i1, llvm::MVT::v8i16, and llvm::MVT::v8i32.
◆ LowerUINT_TO_FP_i32()
|
static |
32-bit unsigned integer to float expansion.
Definition at line 17498 of file X86ISelLowering.cpp.
References llvm::BitsToDouble(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getFPExtendOrRound(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), getShuffleVectorZeroOrUndef(), llvm::SDValue::getSimpleValueType(), llvm::SPII::Load, llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v2f64, llvm::MVT::v2i64, and llvm::MVT::v4i32.
Referenced by lowerUINT_TO_FP_vec().
◆ LowerUINT_TO_FP_i64()
|
static |
64-bit unsigned integer to double expansion.
Definition at line 17432 of file X86ISelLowering.cpp.
References llvm::lltok::APFloat, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::ISD::FSUB, llvm::ConstantFP::get(), llvm::ConstantVector::get(), llvm::ConstantDataVector::get(), llvm::SelectionDAG::getBitcast(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetLoweringInfo(), getUnpackl(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE3(), llvm::APFloatBase::IEEEdouble(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v2f64, llvm::MVT::v2i64, and llvm::MVT::v4i32.
Referenced by lowerUINT_TO_FP_vec().
◆ lowerUINT_TO_FP_v2i32()
|
static |
Definition at line 17535 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, llvm::ISD::FADD, llvm::ISD::FMUL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX512(), llvm::AArch64CC::HI, llvm::AArch64CC::LO, llvm::ISD::SRL, llvm::MVT::v2f64, llvm::MVT::v2i32, and llvm::MVT::v4i32.
Referenced by lowerUINT_TO_FP_vec().
◆ lowerUINT_TO_FP_vec()
|
static |
Definition at line 17665 of file X86ISelLowering.cpp.
References llvm::MCID::Add, llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_PAIR, llvm::X86TargetLowering::BuildFILD(), llvm::APFloat::convert(), llvm::MachineFrameInfo::CreateStackObject(), llvm::SelectionDAG::CreateStackTemporary(), llvm::ISD::EXTLOAD, llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::f80, llvm::ISD::FADD, llvm::X86ISD::FILD, llvm::X86ISD::FLD, llvm::ISD::FP_ROUND, llvm::X86ISD::FP_TO_INT16_IN_MEM, llvm::X86ISD::FP_TO_INT32_IN_MEM, llvm::X86ISD::FP_TO_INT64_IN_MEM, llvm::ISD::FSUB, llvm::ConstantInt::get(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExtLoad(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::X86TargetLowering::getSetCCResultType(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::X86Subtarget::is64Bit(), llvm::X86TargetLowering::isScalarFPTypeInSSEReg(), llvm::MVT::isVector(), LLVM_ATTRIBUTE_UNUSED, llvm_unreachable, LowerI64IntToFP_AVX512DQ(), LowerUINT_TO_FP_i32(), LowerUINT_TO_FP_i64(), lowerUINT_TO_FP_v2i32(), lowerUINT_TO_FP_vXi32(), llvm::MachineMemOperand::MOLoad, llvm::MachineMemOperand::MOStore, llvm::APFloatBase::opOK, llvm::ISD::OR, llvm::MVT::Other, llvm::APFloatBase::rmNearestTiesToEven, llvm::ISD::SETLT, llvm::ISD::SHL, llvm::MVT::SimpleTy, llvm::SPII::Store, llvm::MVT::v2i32, llvm::MVT::v4i32, llvm::MVT::v8i32, llvm::APFloatBase::x87DoubleExtended(), llvm::ISD::XOR, llvm::ISD::ZERO_EXTEND, and llvm::APInt::zext().
◆ lowerUINT_TO_FP_vXi32()
|
static |
Definition at line 17571 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::lltok::APFloat, assert(), llvm::X86ISD::BLENDI, llvm::ISD::FADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTarget(), llvm::X86Subtarget::hasSSE41(), High, llvm::MVT::i32, llvm::APFloatBase::IEEEsingle(), llvm::TargetMachine::Options, llvm::ISD::OR, llvm::ISD::SRL, llvm::TargetOptions::UnsafeFPMath, llvm::MVT::v16i16, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v8f32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by lowerUINT_TO_FP_vec().
◆ lowerV16F32VectorShuffle()
|
static |
Handle lowering of 16-lane 32-bit floating point shuffles.
Definition at line 15457 of file X86ISelLowering.cpp.
References assert(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, llvm::MVT::v16i32, llvm::X86ISD::VPERMILPI, and llvm::X86ISD::VPERMILPV.
Referenced by lower512BitVectorShuffle().
◆ lowerV16I16VectorShuffle()
|
static |
Handle lowering of 16-lane 16-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v16i16 shuffling..
Definition at line 15038 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsLanePermuteAndPermute(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleWithPACK(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithPSHUFB(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), and llvm::MVT::v16i16.
Referenced by lower256BitVectorShuffle().
◆ lowerV16I32VectorShuffle()
|
static |
Handle lowering of 16-lane 32-bit integer shuffles.
Definition at line 15578 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasBWI(), is128BitLaneRepeatedShuffleMask(), isSingleSHUFPSMask(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, and llvm::MVT::v16i32.
Referenced by lower512BitVectorShuffle().
◆ lowerV16I8VectorShuffle()
|
static |
Generic lowering of v16i8 shuffles.
This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.
Definition at line 13357 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::array_pod_sort(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), canLowerByDroppingEvenElements(), llvm::copy_if(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasVBMI(), llvm::X86Subtarget::hasVLX(), I, llvm::SDValue::isUndef(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendOfPSHUFBs(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsByteRotateAndPermute(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPermuteAndUnpack(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithPACK(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUNPCK(), llvm::BitmaskEnumDetail::Mask(), llvm::none_of(), llvm::X86ISD::PACKUS, llvm::X86ISD::PSHUFB, llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, and llvm::MVT::v8i16.
Referenced by lower128BitVectorShuffle().
◆ lowerV2F64VectorShuffle()
|
static |
Handle lowering of 2-lane 64-bit floating point shuffles.
This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.
Definition at line 12082 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i8, isShuffleEquivalent(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVSD, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::MVT::v2f64, and llvm::X86ISD::VPERMILPI.
Referenced by lower128BitVectorShuffle().
◆ lowerV2I64VectorShuffle()
|
static |
Handle lowering of 2-lane 64-bit integer shuffles.
Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by the integer unit to minimize domain crossing penalties. However, for blends it falls back to the floating point shuffle operation with appropriate bit casting.
Definition at line 12164 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasVLX(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleWithUNPCK(), llvm::max(), llvm::X86ISD::PSHUFD, llvm::ArrayRef< T >::size(), llvm::MVT::v2f64, llvm::MVT::v2i64, and llvm::MVT::v4i32.
Referenced by lower128BitVectorShuffle().
◆ lowerV2X128VectorShuffle()
|
static |
Handle lowering 2-lane 128-bit shuffles.
Definition at line 13993 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, isShuffleEquivalent(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), llvm::peekThroughBitcasts(), llvm::X86ISD::SHUF128, and llvm::X86ISD::VPERM2X128.
Referenced by lowerV4F64VectorShuffle(), and lowerV4I64VectorShuffle().
◆ lowerV32I16VectorShuffle()
|
static |
Handle lowering of 32-lane 16-bit integer shuffles.
Definition at line 15649 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasBWI(), is128BitLaneRepeatedShuffleMask(), llvm::SDValue::isUndef(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithPSHUFB(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), and llvm::MVT::v32i16.
Referenced by lower512BitVectorShuffle().
◆ lowerV32I8VectorShuffle()
|
static |
Handle lowering of 32-lane 8-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v32i8 shuffling..
Definition at line 15140 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasVBMI(), llvm::X86Subtarget::hasVLX(), is128BitLaneCrossingShuffleMask(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsLanePermuteAndPermute(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleWithPACK(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithPSHUFB(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), and llvm::MVT::v32i8.
Referenced by lower256BitVectorShuffle().
◆ lowerV4F32VectorShuffle()
|
static |
Lower 4-lane 32-bit floating point shuffles.
Uses instructions exclusively from the floating point unit to minimize domain crossing penalties, as these are sufficient to implement all v4f32 shuffles.
Definition at line 12369 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendAndPermute(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsInsertPS(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::MVT::v4f32, and llvm::X86ISD::VPERMILPI.
Referenced by lower128BitVectorShuffle().
◆ lowerV4F64VectorShuffle()
|
static |
Handle lowering of 4-lane 64-bit floating point shuffles.
Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2 isn't available.
Definition at line 14627 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), isShuffleEquivalent(), isShuffleMaskInputInPlace(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV2X128VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsLanePermuteAndPermute(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithSHUFPD(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVDDUP, llvm::ArrayRef< T >::size(), llvm::MVT::v4f64, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by lower256BitVectorShuffle().
◆ lowerV4I32VectorShuffle()
|
static |
Lower 4-lane i32 vector shuffles.
We try to handle these with integer-domain shuffles where we can, but for blends we use the floating point domain blend instructions.
Definition at line 12464 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasVLX(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPermuteAndUnpack(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, llvm::ArrayRef< T >::size(), llvm::MVT::v4f32, and llvm::MVT::v4i32.
Referenced by lower128BitVectorShuffle().
◆ lowerV4I64VectorShuffle()
|
static |
Handle lowering of 4-lane 64-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v4i64 shuffling..
Definition at line 14731 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasVLX(), is128BitLaneRepeatedShuffleMask(), isShuffleMaskInputInPlace(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV2X128VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, llvm::ArrayRef< T >::size(), llvm::MVT::v4i64, llvm::MVT::v8i32, and llvm::X86ISD::VPERMI.
Referenced by lower256BitVectorShuffle().
◆ lowerV4X128VectorShuffle()
|
static |
Try to lower a vector shuffle as a 128-bit shuffles.
Definition at line 15298 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i8, insert128BitVector(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), isShuffleEquivalent(), isUndef(), llvm::X86ISD::SHUF128, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV8F64VectorShuffle(), and lowerV8I64VectorShuffle().
◆ lowerV64I8VectorShuffle()
|
static |
Handle lowering of 64-lane 8-bit integer shuffles.
Definition at line 15704 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVBMI(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithPACK(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithPSHUFB(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), splitAndLowerVectorShuffle(), and llvm::MVT::v64i8.
Referenced by lower512BitVectorShuffle().
◆ lowerV8F32VectorShuffle()
|
static |
Handle lowering of 8-lane 32-bit floating point shuffles.
Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2 isn't available.
Definition at line 14825 of file X86ISelLowering.cpp.
References assert(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), llvm::SDValue::isUndef(), isUnpackWdShuffleMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::MVT::v8f32, llvm::MVT::v8i32, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, and llvm::X86ISD::VPERMV.
Referenced by lower256BitVectorShuffle().
◆ lowerV8F64VectorShuffle()
|
static |
Handle lowering of 8-lane 64-bit floating point shuffles.
Definition at line 15400 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), is256BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), llvm::SDValue::isUndef(), lowerV4X128VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithSHUFPD(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVDDUP, llvm::ArrayRef< T >::size(), llvm::MVT::v8f64, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by lower512BitVectorShuffle().
◆ lowerV8I16GeneralSingleInputVectorShuffle()
|
static |
Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.
The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.
The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.
This code also handles repeated 128-bit lanes of v8i16 shuffles, but each lane must shuffle the exact same way. In fact, you must pass a v8 Mask to this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16 vector, form the analogous 128-bit 8-element Mask.
Definition at line 12588 of file X86ISelLowering.cpp.
References llvm::array_pod_sort(), assert(), llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::MutableArrayRef< T >::begin(), llvm::copy_if(), llvm::count(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::data(), llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::MutableArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::find(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::is_contained(), isNoopShuffleMask(), isSequentialOrUndefInRange(), isUndefOrEqual(), isUndefOrInRange(), llvm_unreachable, llvm::lower_bound(), llvm::BitmaskEnumDetail::Mask(), llvm::Match, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T >::push_back(), replace(), llvm::SmallVectorImpl< T >::resize(), llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::MutableArrayRef< T >::slice(), and std::swap().
Referenced by lowerV16I16VectorShuffle(), lowerV32I16VectorShuffle(), and lowerV8I16VectorShuffle().
◆ lowerV8I16VectorShuffle()
|
static |
Generic lowering of 8-lane i16 shuffles.
This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.
The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.
Definition at line 13150 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::count_if(), llvm::ArrayRef< T >::end(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendOfPSHUFBs(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPermuteAndUnpack(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithPACK(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUNPCK(), llvm::ArrayRef< T >::size(), and llvm::MVT::v8i16.
Referenced by lower128BitVectorShuffle().
◆ lowerV8I32VectorShuffle()
|
static |
Handle lowering of 8-lane 32-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v8i32 shuffling..
Definition at line 14925 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), is128BitLaneRepeatedShuffleMask(), isSingleSHUFPSMask(), llvm::SDValue::isUndef(), isUnpackWdShuffleMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::MVT::v8f32, llvm::MVT::v8i32, and llvm::X86ISD::VPERMV.
Referenced by lower256BitVectorShuffle().
◆ lowerV8I64VectorShuffle()
|
static |
Handle lowering of 8-lane 64-bit integer shuffles.
Definition at line 15512 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneRepeatedShuffleMask(), is256BitLaneRepeatedShuffleMask(), llvm::SDValue::isUndef(), lowerV4X128VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, llvm::ArrayRef< T >::size(), llvm::MVT::v16i32, llvm::MVT::v8i64, and llvm::X86ISD::VPERMI.
Referenced by lower512BitVectorShuffle().
◆ LowerVACOPY()
|
static |
Definition at line 21178 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::expandVACopy(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemcpy(), llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::X86Subtarget::is64Bit(), and llvm::X86Subtarget::isCallingConvWin64().
Referenced by llvm::PPCTargetLowering::functionArgumentNeedsConsecutiveRegisters(), llvm::X86TargetLowering::LowerOperation(), and llvm::AArch64TargetLowering::supportSwiftError().
◆ LowerVectorAllZeroTest()
|
static |
Definition at line 18745 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T >::back(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::begin(), llvm::SmallVectorTemplateCommon< T >::begin(), llvm::X86::COND_E, llvm::X86::COND_NE, E, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE41(), I, llvm::MVT::i32, llvm::MVT::i8, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::ISD::OR, llvm::MVT::Other, llvm::X86ISD::PTEST, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SETEQ, llvm::SmallVectorBase::size(), llvm::MVT::v2i64, and llvm::MVT::v4i64.
Referenced by EmitKORTEST().
◆ LowerVectorCTLZ()
|
static |
Definition at line 23385 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasCDI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), Lower256IntUnary(), Lower512IntUnary(), LowerVectorCTLZ_AVX512CDI(), and LowerVectorCTLZInRegLUT().
Referenced by LowerCTLZ().
◆ LowerVectorCTLZ_AVX512CDI()
|
static |
Lower a vector CTLZ using native supported vector CTLZ instruction.
Definition at line 23273 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerVectorIntUnary(), llvm::ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerVectorCTLZ().
◆ LowerVectorCTLZInRegLUT()
|
static |
Definition at line 23303 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i8, llvm::MipsISD::Lo, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, and llvm::ISD::SRL.
Referenced by LowerVectorCTLZ().
◆ LowerVectorCTPOP()
|
static |
Definition at line 25534 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::CTPOP, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasVPOPCNTDQ(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), Lower256IntUnary(), Lower512IntUnary(), LowerHorizontalByteSum(), LowerVectorCTPOPInRegLUT(), llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerCTPOP().
◆ LowerVectorCTPOPInRegLUT()
|
static |
Definition at line 25487 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::i8, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T >::push_back(), and llvm::ISD::SRL.
Referenced by LowerVectorCTPOP().
◆ LowerVectorIntUnary()
|
static |
Definition at line 23231 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MipsISD::Hi, and llvm::MipsISD::Lo.
Referenced by Lower256IntUnary(), Lower512IntUnary(), and LowerVectorCTLZ_AVX512CDI().
◆ lowerVectorShuffle()
|
static |
Top-level lowering for x86 vector shuffles.
This handles decomposition, canonicalization, and lowering of all x86 vector shuffles. Most of the specific lowering strategies are encapsulated above in helper routines. The canonicalization attempts to widen shuffles to involve fewer lanes of wider elements, consolidate symmetric patterns s.t. only one of the two inputs needs to be tested, etc.
Definition at line 16021 of file X86ISelLowering.cpp.
References llvm::all_of(), assert(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), computeZeroableShuffleElements(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::find(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCommutedVectorShuffle(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i1, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::APInt::isAllOnesValue(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), llvm::APInt::isNullValue(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm_unreachable, lower128BitVectorShuffle(), lower1BitVectorShuffle(), lower256BitVectorShuffle(), lower512BitVectorShuffle(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleWithVPMOV(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by llvm::X86TargetLowering::LowerOperation().
◆ lowerVectorShuffleAsBitBlend()
|
static |
Try to emit a blend instruction for a shuffle using bit math.
This is used as a fallback approach when first class blend instructions are unavailable. Currently it is only suitable for integer vectors, but could be generalized for floating point vectors if desirable.
Definition at line 10224 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), getVectorMaskingNode(), llvm::MVT::isInteger(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T >::push_back(), Size, and llvm::ArrayRef< T >::size().
Referenced by lower256BitVectorShuffle(), lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
◆ lowerVectorShuffleAsBitMask()
|
static |
Try to emit a bitmask instruction for a shuffle.
This handles cases where we can model a blend exactly as a bitmask due to one of the inputs being zeroable.
Definition at line 10190 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::isFloatingPoint(), Size, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lower256BitVectorShuffle(), lowerV16I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerVectorShuffleAsBlend().
◆ lowerVectorShuffleAsBlend()
|
static |
Try to emit a blend instruction for a shuffle.
This doesn't do any checks for the availability of instructions for blending these values. It relies on the availability of the X86ISD::BLENDI pattern to be matched in the backend with the type given. What it does check for is that the shuffle mask is a blend, or convertible into a blend with zero.
Definition at line 10308 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BLENDI, llvm::ShuffleVectorSDNode::commuteMask(), createTargetShuffleMask(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getVectorMaskingNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MipsISD::Hi, llvm::MVT::i8, is128BitLaneRepeatedShuffleMask(), llvm::MVT::is128BitVector(), llvm::ISD::isNormalLoad(), LLVM_FALLTHROUGH, llvm_unreachable, llvm::MipsISD::Lo, lowerVectorShuffleAsBitMask(), matchVectorShuffleAsBlend(), llvm::max(), llvm::SmallVectorTemplateBase< T >::push_back(), scaleVectorShuffleBlendMask(), llvm::MVT::SimpleTy, Size, llvm::SmallVectorBase::size(), std::swap(), llvm::MVT::v16f32, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v64i8, llvm::MVT::v8f32, llvm::MVT::v8f64, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::MVT::v8i64.
Referenced by lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
◆ lowerVectorShuffleAsBlendAndPermute()
|
static |
Try to lower as a blend of elements from two inputs followed by a single-input permutation.
This matches the pattern where we can blend elements from two inputs and then reduce the shuffle to a single-input permutation.
Definition at line 10474 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), Size, and llvm::ArrayRef< T >::size().
Referenced by lowerV4F32VectorShuffle(), and lowerVectorShuffleAsDecomposedShuffleBlend().
◆ lowerVectorShuffleAsBlendOfPSHUFBs()
|
static |
Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used.
Definition at line 13087 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorVT(), llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), llvm::ISD::OR, llvm::X86ISD::PSHUFB, Size, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
◆ lowerVectorShuffleAsBroadcast()
|
static |
Try to lower broadcast of a single element.
For convenience, this code also bundles all of the subtarget feature set filtering. While a little annoying to re-dispatch on type here, there isn't a convenient way to factor it out.
Definition at line 11637 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), extract128BitVector(), llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::MVT::getSizeInBits(), llvm::EVT::getStoreSize(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ConstantSDNode::getZExtValue(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), llvm::MVT::i64, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm::MVT::isInteger(), isShuffleEquivalent(), isShuffleFoldableLoad(), llvm::MVT::isVector(), isVolatile(), lowerVectorShuffleAsTruncBroadcast(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), MayFoldLoad(), llvm::X86ISD::MOVDDUP, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ArrayRef< T >::size(), llvm::MVT::v2f64, llvm::MVT::v4f64, llvm::MVT::v4i64, and llvm::X86ISD::VBROADCAST.
Referenced by lower512BitVectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerVectorShuffle().
◆ lowerVectorShuffleAsByteRotate()
|
static |
Definition at line 10821 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is512BitVector(), isNoopShuffleMask(), llvm::MipsISD::Lo, matchVectorShuffleAsByteRotate(), llvm::ISD::OR, llvm::X86ISD::PALIGNR, llvm::ArrayRef< T >::size(), llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
Referenced by lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
◆ lowerVectorShuffleAsByteRotateAndPermute()
|
static |
Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place.
Definition at line 10576 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MipsISD::Lo, llvm::max(), llvm::X86ISD::PALIGNR, and llvm::SM_SentinelUndef.
Referenced by lowerV16I8VectorShuffle(), and lowerVectorShuffleAsDecomposedShuffleBlend().
◆ lowerVectorShuffleAsDecomposedShuffleBlend()
|
static |
Generic routine to decompose a shuffle and blend into independent blends and permutes.
This matches the extremely common pattern for handling combined shuffle+blend operations on newer X86 ISAs where we have very fast blend operations. It will try to pick the best arrangement of shuffles and blends.
Definition at line 10667 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), isNoopShuffleMask(), lowerVectorShuffleAsBlendAndPermute(), lowerVectorShuffleAsByteRotateAndPermute(), lowerVectorShuffleAsUNPCKAndPermute(), Size, and llvm::ArrayRef< T >::size().
Referenced by lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerVectorShuffleAsSplitOrBlend().
◆ lowerVectorShuffleAsElementInsertion()
|
static |
Try to lower insertion of a single element into a zero vector.
This is a common pattern that we have especially efficient patterns to lower across all subtarget feature sets.
Definition at line 11485 of file X86ISelLowering.cpp.
References assert(), llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::end(), llvm::MVT::f32, llvm::MVT::f64, llvm::find_if(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::ISD::SCALAR_TO_VECTOR, Size, llvm::ArrayRef< T >::size(), llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VZEXT_MOVL, and llvm::ISD::ZERO_EXTEND.
Referenced by lower256BitVectorShuffle(), lower512BitVectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().
◆ lowerVectorShuffleAsInsertPS()
|
static |
Definition at line 11941 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::i8, llvm::X86ISD::INSERTPS, matchVectorShuffleAsInsertPS(), and llvm::MVT::v4f32.
Referenced by lowerV4F32VectorShuffle().
◆ lowerVectorShuffleAsLanePermuteAndBlend()
|
static |
Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes.
This essentially blends the out-of-lane inputs to each lane into the lane from a permuted copy of the vector. This lowering strategy results in four instructions in the worst case for a single-input cross lane shuffle which is lower than any other fully general cross-lane shuffle strategy I'm aware of. Special cases for each particular shuffle pattern should be handled prior to trying this lowering.
Definition at line 13943 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::is256BitVector(), llvm::MVT::isFloatingPoint(), llvm::SDValue::isUndef(), Size, llvm::ArrayRef< T >::size(), splitAndLowerVectorShuffle(), llvm::MVT::v4f64, and llvm::MVT::v4i64.
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().
◆ lowerVectorShuffleAsLanePermuteAndPermute()
|
static |
Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation.
This is mainly for cases where we can have non-repeating permutes in each lane.
TODO: This is very similar to lowerVectorShuffleByMerging128BitLanes, we should investigate merging them.
Definition at line 13879 of file X86ISelLowering.cpp.
References llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), isSequentialOrUndefInRange(), isUndefOrEqual(), and llvm::SM_SentinelUndef.
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), and lowerV4F64VectorShuffle().
◆ lowerVectorShuffleAsPermuteAndUnpack()
|
static |
Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.
This specifically targets cases where we end up with alternating between the two inputs, and so can permute them into something that feeds a single UNPCK instruction. Note that this routine only targets integer vectors because for floating point vectors we have a generalized SHUFPS lowering strategy that handles everything that doesn't exactly match an unpack, making this clever lowering unnecessary.
Definition at line 11967 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::SDValue::isUndef(), Size, llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerV16I8VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().
◆ lowerVectorShuffleAsRotate()
|
static |
Try to lower a vector shuffle as a dword/qword rotation.
AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary rotation of the concatenation of two vectors; This routine will try to generically lower a vector shuffle through such an pattern.
Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.
Definition at line 10877 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::X86Subtarget::hasVLX(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MipsISD::Lo, matchVectorShuffleAsRotate(), llvm::NVPTX::PTXLdStInstCode::V2, and llvm::X86ISD::VALIGN.
Referenced by lowerV16I32VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
◆ lowerVectorShuffleAsShift()
|
static |
Definition at line 10984 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::MVT::i8, llvm::TargetLoweringBase::isTypeLegal(), llvm::BitmaskEnumDetail::Mask(), matchVectorShuffleAsShift(), Size, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
◆ lowerVectorShuffleAsSpecificZeroOrAnyExtend()
|
static |
Lower a vector shuffle as a zero or any extension.
Given a specific number of elements, element bit width, and extension stride, produce either a zero or any extension based on the available features of the subtarget. The extended elements are consecutive and begin and can start from an offsetted element index in the input; to avoid excess shuffling the offset must either being in the bottom lane or at the start of a higher lane. All extended elements must be from the same lane.
Definition at line 11171 of file X86ISelLowering.cpp.
References assert(), llvm::MipsISD::Ext, llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getExtendInVec(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::MVT::is128BitVector(), isUndefInRange(), llvm::MipsISD::Lo, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, and llvm::MVT::v8i16.
Referenced by lowerVectorShuffleAsZeroOrAnyExtend().
◆ lowerVectorShuffleAsSplitOrBlend()
|
static |
Either split a vector in halves or decompose the shuffles and the blend.
This is provided as a good fallback for many lowerings of non-single-input shuffles with more than one 128-bit lane. In those cases, we want to select between splitting the shuffle into 128-bit components and stitching those back together vs. extracting the single-input shuffles and blending those results.
Definition at line 13819 of file X86ISelLowering.cpp.
References assert(), llvm::SmallBitVector::count(), llvm::count(), llvm::MVT::getSizeInBits(), llvm::SDValue::isUndef(), lowerVectorShuffleAsDecomposedShuffleBlend(), llvm::SmallBitVector::resize(), Size, llvm::ArrayRef< T >::size(), and splitAndLowerVectorShuffle().
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
◆ lowerVectorShuffleAsTruncBroadcast()
|
static |
Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements.
This assumes we have AVX2.
Definition at line 11583 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, llvm::MVT::isInteger(), llvm::EVT::isVector(), llvm::NVPTX::PTXLdStInstCode::Scalar, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::X86ISD::VBROADCAST.
Referenced by lowerVectorShuffleAsBroadcast().
◆ lowerVectorShuffleAsUNPCKAndPermute()
|
static |
Try to lower as an unpack of elements from two inputs followed by a single-input permutation.
This matches the pattern where we can unpack elements from two inputs and then reduce the shuffle to a single-input (wider) permutation.
Definition at line 10513 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MipsISD::Hi, llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::MipsISD::Lo, llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerVectorShuffleAsDecomposedShuffleBlend().
◆ lowerVectorShuffleAsZeroOrAnyExtend()
|
static |
Try to lower a vector shuffle as a zero extension on any microarch.
This routine will try to do everything in its power to cleverly lower a shuffle which happens to match the pattern of a zero extend. It doesn't check for the profitability of this lowering, it tries to aggressively match this pattern. It will use all of the micro-architectural details it can to emit an efficient lowering. It handles both blends with all-zero inputs to explicitly zero-extend and undef-lanes (sometimes undef due to masking out later).
The reason we have dedicated lowering for zext-style shuffles is that they are both incredibly common and often quite performance sensitive.
Definition at line 11329 of file X86ISelLowering.cpp.
References assert(), llvm::tgtok::Bits, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), isSequentialOrUndefInRange(), llvm::Lower, lowerVectorShuffleAsSpecificZeroOrAnyExtend(), llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, and llvm::X86ISD::VZEXT_MOVL.
Referenced by lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
◆ lowerVectorShuffleByMerging128BitLanes()
|
static |
Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.
This attempts to create a repeated lane shuffle where each lane uses one or two of the lanes of the inputs. The lanes of the input vectors are shuffled in one or two independent shuffles to get the lanes into the position needed by the final shuffle.
FIXME: This should be generalized to 512-bit shuffles.
Definition at line 14098 of file X86ISelLowering.cpp.
References assert(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getVectorShuffle(), is128BitLaneRepeatedShuffleMask(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), Size, llvm::ArrayRef< T >::size(), and std::swap().
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
◆ lowerVectorShuffleToEXPAND()
|
static |
Definition at line 9888 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::EXPAND, llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), getMaskNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i1, isNonZeroElementsInOrder(), llvm::max(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
◆ lowerVectorShuffleWithPACK()
|
static |
Definition at line 10172 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), and matchVectorShuffleWithPACK().
Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV32I8VectorShuffle(), lowerV64I8VectorShuffle(), and lowerV8I16VectorShuffle().
◆ lowerVectorShuffleWithPERMV()
|
static |
Definition at line 13337 of file X86ISelLowering.cpp.
References getConstVector(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::isUndef(), llvm::X86ISD::VPERMV, and llvm::X86ISD::VPERMV3.
Referenced by lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F64VectorShuffle(), and lowerV8I64VectorShuffle().
◆ lowerVectorShuffleWithPSHUFB()
|
static |
Try to lower a shuffle with a single PSHUFB of V1 or V2.
Definition at line 9829 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getMaskNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::X86ISD::PSHUFB, Size, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I16VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), and lowerV64I8VectorShuffle().
◆ lowerVectorShuffleWithSHUFPD()
|
static |
Definition at line 14609 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i8, matchVectorShuffleWithSHUFPD(), llvm::X86ISD::SHUFP, llvm::MVT::v2f64, llvm::MVT::v4f64, and llvm::MVT::v8f64.
Referenced by lowerV4F64VectorShuffle(), and lowerV8F64VectorShuffle().
◆ lowerVectorShuffleWithSHUFPS()
|
static |
Lower a vector shuffle using the SHUFPS instruction.
This is a helper routine dedicated to lowering vector shuffles using SHUFPS. It makes no assumptions about whether this is the best lowering, it simply uses it.
Definition at line 12282 of file X86ISelLowering.cpp.
References llvm::count_if(), llvm::find_if(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::X86ISD::SHUFP, std::swap(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV4F32VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
◆ lowerVectorShuffleWithSSE4A()
|
static |
Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
Definition at line 11143 of file X86ISelLowering.cpp.
References llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::i8, llvm::X86ISD::INSERTQI, matchVectorShuffleAsEXTRQ(), matchVectorShuffleAsINSERTQ(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
◆ lowerVectorShuffleWithUndefHalf()
|
static |
Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
This allows for fast cases such as subvector extraction/insertion or shuffling smaller vector types which can lower more efficiently.
Definition at line 14265 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MipsISD::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isSequentialOrUndefInRange(), isUndefInRange(), llvm::SmallVectorBase::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v8f32, and llvm::MVT::v8i32.
Referenced by lower256BitVectorShuffle(), and lower512BitVectorShuffle().
◆ lowerVectorShuffleWithUNPCK()
|
static |
Definition at line 10000 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::commuteMask(), llvm::createUnpackShuffleMask(), llvm::SelectionDAG::getNode(), isShuffleEquivalent(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
◆ lowerVectorShuffleWithVPMOV()
|
static |
Definition at line 10064 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i16, llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), matchVectorShuffleAsVPMOV(), llvm::ArrayRef< T >::size(), std::swap(), llvm::ISD::TRUNCATE, llvm::MVT::v16i8, llvm::MVT::v8i16, and llvm::X86ISD::VTRUNC.
Referenced by lowerVectorShuffle().
◆ lowerVSELECTtoVectorShuffle()
|
static |
Try to lower a VSELECT instruction to a vector shuffle.
Definition at line 16159 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::ComputeNumSignBits(), createShuffleMaskFromVSELECT(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i1, llvm::MVT::i8, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::BitmaskEnumDetail::Mask(), llvm::MCID::Select, llvm::ISD::SETNE, llvm::MVT::SimpleTy, llvm::MVT::v16i16, llvm::MVT::v32i8, llvm::MVT::v8i16, and llvm::ISD::VSELECT.
◆ LowerVSETCC()
|
static |
Definition at line 19375 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::AND, assert(), C, llvm::X86ISD::CMPM, llvm::X86ISD::CMPP, EQ, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::AArch64CC::GT, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE42(), llvm::X86Subtarget::hasXOP(), llvm::MVT::i1, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorAllZeros(), isConstantSplat(), llvm::ISD::isConstantSplatVector(), llvm::MVT::isFloatingPoint(), llvm::APInt::isMaxSignedValue(), llvm::APInt::isMaxValue(), llvm::APInt::isMinSignedValue(), llvm::APInt::isNullValue(), llvm::TargetLoweringBase::isOperationLegal(), llvm::APInt::isPowerOf2(), llvm::ISD::isTrueWhenEqual(), llvm::ISD::isUnsignedIntSetCC(), LLVM_FALLTHROUGH, llvm_unreachable, Lower256IntVSETCC(), LowerIntVSETCC_AVX512(), LowerVSETCCWithSUBUS(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::SelectionDAG::SignBitIsZero(), std::swap(), translateX86FSETCC(), llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::X86ISD::VPCOM, llvm::X86ISD::VPCOMU, and llvm::ISD::XOR.
Referenced by combineSetCC(), and EmitKORTEST().
◆ LowerVSETCCWithSUBUS()
|
static |
As another special case, use PSUBUS[BW] when it's profitable.
E.g. for Op0 u<= Op1: t = psubus Op0, Op1 pcmpeq t, <0..0>
Definition at line 19334 of file X86ISelLowering.cpp.
References decrementVectorConstant(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i8, llvm::X86ISD::PCMPEQ, llvm::ISD::SETUGE, llvm::ISD::SETULE, llvm::ISD::SETULT, std::swap(), and llvm::ISD::USUBSAT.
Referenced by LowerVSETCC().
◆ lowerX86CmpEqZeroToCtlzSrl()
|
static |
Definition at line 36590 of file X86ISelLowering.cpp.
References llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::i32, llvm::MVT::i8, llvm::Log2_32(), and llvm::ISD::SRL.
Referenced by combineOrCmpEqZeroToCtlzSrl().
◆ lowerX86FPLogicOp()
|
static |
Definition at line 38708 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::isVector(), llvm_unreachable, llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by combineFAnd(), combineFAndn(), and combineFOr().
◆ LowerXALUO()
|
static |
Definition at line 19868 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), getSETCC(), llvm::SDNode::getVTList(), getX86XALUOOp(), and llvm::ISD::MERGE_VALUES.
Referenced by isXor1OfSetCC(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerZERO_EXTEND()
|
static |
Definition at line 18095 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), llvm::MVT::i1, llvm::tgtok::In, LowerAVXExtend(), and LowerZERO_EXTEND_Mask().
Referenced by llvm::HexagonTargetLowering::isCtlzFast(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerZERO_EXTEND_Mask()
|
static |
Definition at line 18037 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, llvm::tgtok::In, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), llvm::ISD::SIGN_EXTEND, SplitAndExtendv16i1(), llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerZERO_EXTEND().
◆ MarkEHGuard()
|
static |
Definition at line 22577 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::WinEHFuncInfo::EHGuardFrameIndex, llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getOperand(), llvm::MachineFunction::getWinEHFuncInfo(), and llvm::report_fatal_error().
Referenced by LowerINTRINSIC_W_CHAIN().
◆ MarkEHRegistrationNode()
|
static |
Definition at line 22559 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::WinEHFuncInfo::EHRegNodeFrameIndex, llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getOperand(), llvm::MachineFunction::getWinEHFuncInfo(), and llvm::report_fatal_error().
Referenced by LowerINTRINSIC_W_CHAIN().
◆ match1BitShuffleAsKSHIFT()
|
static |
Definition at line 15822 of file X86ISelLowering.cpp.
References isSequentialOrUndefInRange(), llvm::X86ISD::KSHIFTL, llvm::X86ISD::KSHIFTR, llvm::Left, Size, and llvm::ArrayRef< T >::size().
Referenced by lower1BitVectorShuffle().
◆ matchAsm()
Definition at line 41796 of file X86ISelLowering.cpp.
References llvm::StringRef::empty(), llvm::StringRef::find_first_not_of(), llvm::StringRef::startswith(), and llvm::StringRef::substr().
Referenced by llvm::X86TargetLowering::ExpandInlineAsm().
◆ matchBinaryPermuteVectorShuffle()
|
static |
Definition at line 30645 of file X86ISelLowering.cpp.
References assert(), llvm::ArrayRef< T >::begin(), llvm::X86ISD::BLENDI, llvm::ArrayRef< T >::end(), llvm::MVT::f32, llvm::MVT::f64, llvm::APInt::getBoolValue(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), isRepeatedTargetShuffleMask(), isUndefInRange(), isUndefOrInRange(), isUndefOrZeroInRange(), llvm::MipsISD::Lo, matchVectorShuffleAsBlend(), matchVectorShuffleAsByteRotate(), matchVectorShuffleAsInsertPS(), matchVectorShuffleWithSHUFPD(), llvm::X86ISD::PALIGNR, scaleVectorShuffleBlendMask(), llvm::X86ISD::SHUFP, llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::MVT::v16i16, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8f32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by combineX86ShuffleChain().
◆ matchBinaryVectorShuffle()
|
static |
Definition at line 30578 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isTargetShuffleEquivalent(), matchVectorShuffleWithPACK(), matchVectorShuffleWithUNPCK(), llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::SM_SentinelUndef, std::swap(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v64i8, llvm::MVT::v8f32, and llvm::MVT::v8i16.
Referenced by combineX86ShuffleChain().
◆ MatchingStackOffset()
|
static |
Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack.
Definition at line 4212 of file X86ISelLowering.cpp.
References llvm::CCState::AllocateStack(), llvm::CCState::AnalyzeCallOperands(), llvm::CCState::AnalyzeCallResult(), llvm::ISD::ANY_EXTEND, Arg, assert(), llvm::ISD::AssertZext, llvm::ISD::BITCAST, C, Callee, canGuaranteeTCO(), llvm::ISD::CopyFromReg, llvm::tgtok::Def, llvm::dyn_cast(), llvm::N86::EAX, llvm::N86::ECX, llvm::N86::EDX, llvm::SmallVectorBase::empty(), llvm::ISD::FrameIndex, llvm::X86MachineFunctionInfo::getBytesToPopOnReturn(), llvm::ISD::ArgFlagsTy::getByValSize(), llvm::Function::getCallingConv(), llvm::X86RegisterInfo::getCallPreservedMask(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFrameInfo(), llvm::MachineFunction::getFunction(), llvm::MachineOperand::getIndex(), llvm::FrameIndexSDNode::getIndex(), llvm::MachineFunction::getInfo(), llvm::X86Subtarget::getInstrInfo(), llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocMemOffset(), llvm::CCValAssign::getLocReg(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::CCState::getNextStackOffset(), llvm::MachineFrameInfo::getObjectOffset(), llvm::MachineFrameInfo::getObjectSize(), llvm::SDValue::getOpcode(), llvm::MachineInstr::getOpcode(), llvm::SDValue::getOperand(), llvm::MachineInstr::getOperand(), getReg(), llvm::MachineFunction::getRegInfo(), llvm::X86Subtarget::getRegisterInfo(), llvm::Function::getReturnType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTarget(), llvm::MachineFunction::getTarget(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MachineRegisterInfo::getVRegDef(), llvm::TargetOptions::GuaranteedTailCallOpt, llvm::CCValAssign::Indirect, llvm::X86Subtarget::is64Bit(), llvm::ISD::ArgFlagsTy::isByVal(), llvm::X86::isCalleePop(), llvm::X86Subtarget::isCallingConvWin64(), llvm::MachineOperand::isFI(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::MachineFrameInfo::isImmutableObjectIndex(), llvm::X86InstrInfo::isLoadFromStackSlot(), llvm::MachineFrameInfo::isObjectSExt(), llvm::MachineFrameInfo::isObjectZExt(), llvm::TargetLowering::isPositionIndependent(), llvm::CCValAssign::isRegLoc(), llvm::ISD::ArgFlagsTy::isSExt(), llvm::TargetRegisterInfo::isVirtualRegister(), llvm::Type::isX86_FP80Ty(), llvm::ISD::ArgFlagsTy::isZExt(), mayTailCallThisCC(), MRI, llvm::TargetMachine::Options, llvm::TargetLowering::parametersInCSRMatch(), Reg, llvm::CCState::resultsCompatible(), llvm::SmallVectorBase::size(), TII, TRI, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
◆ matchLogicBlend()
Definition at line 36455 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::BitmaskEnumDetail::Mask(), llvm::ISD::OR, and std::swap().
Referenced by combineLogicBlendIntoPBLENDV().
◆ matchPMADDWD()
|
static |
Definition at line 40723 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, canReduceVMulWidth(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, llvm::isPowerOf2_32(), llvm::EVT::isVector(), Mode, llvm::ISD::MUL, SplitOpsAndApply(), std::swap(), llvm::ISD::TRUNCATE, and llvm::X86ISD::VPMADDWD.
Referenced by combineAdd().
◆ matchPMADDWD_2()
|
static |
Definition at line 40868 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::ISD::SIGN_EXTEND, SplitOpsAndApply(), std::swap(), and llvm::X86ISD::VPMADDWD.
Referenced by combineAdd().
◆ matchUnaryPermuteVectorShuffle()
|
static |
Definition at line 30456 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), llvm::ArrayRef< T >::data(), llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::getIntegerVT(), llvm::MVT::getSizeInBits(), getV4X86ShuffleImm(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i16, llvm::MVT::i32, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is128BitVector(), is256BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isSequentialOrUndefInRange(), isUndefOrInRange(), matchVectorShuffleAsShift(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v8f64, llvm::MVT::v8i64, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by combineX86ShuffleChain().
◆ matchUnaryVectorShuffle()
|
static |
Definition at line 30319 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isInteger(), isTargetShuffleEquivalent(), isUndefInRange(), isUndefOrEqual(), isUndefOrZero(), isUndefOrZeroInRange(), llvm::Match, llvm::max(), llvm::ScaledNumbers::MaxScale, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, llvm::MVT::v8f64, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VZEXT_MOVL, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by combineX86ShuffleChain().
◆ matchVectorShuffleAsBlend()
|
static |
Definition at line 10249 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), Size, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by lowerVectorShuffleAsBlend(), and matchBinaryPermuteVectorShuffle().
◆ matchVectorShuffleAsByteRotate()
|
static |
Try to lower a vector shuffle as a byte rotation.
SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary byte-rotation of the concatenation of two vectors; pre-SSSE3 can use a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will try to generically lower a vector shuffle through such an pattern. It does not check for the profitability of lowering either as PALIGNR or PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. This matches shuffle vectors that look like:
v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.
Definition at line 10799 of file X86ISelLowering.cpp.
References llvm::any_of(), is128BitLaneRepeatedShuffleMask(), matchVectorShuffleAsRotate(), llvm::SmallVectorBase::size(), and llvm::SM_SentinelZero.
Referenced by lowerVectorShuffleAsByteRotate(), and matchBinaryPermuteVectorShuffle().
◆ matchVectorShuffleAsEXTRQ()
|
static |
Definition at line 11021 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::APInt::isAllOnesValue(), isUndefInRange(), Size, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by combineX86ShuffleChain(), and lowerVectorShuffleWithSSE4A().
◆ matchVectorShuffleAsInsertPS()
|
static |
Definition at line 11852 of file X86ISelLowering.cpp.
References assert(), llvm::ArrayRef< T >::begin(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::ArrayRef< T >::end(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::is128BitVector(), llvm::ArrayRef< T >::size(), llvm::MVT::v4f32, and llvm::codeview::VB.
Referenced by lowerVectorShuffleAsInsertPS(), and matchBinaryPermuteVectorShuffle().
◆ matchVectorShuffleAsINSERTQ()
|
static |
Definition at line 11077 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MipsISD::Hi, isSequentialOrUndefInRange(), isUndefInRange(), Size, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by combineX86ShuffleChain(), and lowerVectorShuffleWithSSE4A().
◆ matchVectorShuffleAsRotate()
Try to lower a vector shuffle as a rotation.
This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
Definition at line 10714 of file X86ISelLowering.cpp.
References assert(), llvm::MipsISD::Hi, llvm::MipsISD::Lo, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerVectorShuffleAsRotate(), and matchVectorShuffleAsByteRotate().
◆ matchVectorShuffleAsShift()
|
static |
Try to lower a vector shuffle as a bit shift (shifts in zeros).
Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function matches elements from one of the input vectors shuffled to the left or right with zeroable elements 'shifted in'. It handles both the strictly bit-wise element shifts and the byte shift across an entire 128-bit double quad word lane.
PSHL : (little-endian) left bit shift. [ zz, 0, zz, 2 ] [ -1, 4, zz, -1 ] PSRL : (little-endian) right bit shift. [ 1, zz, 3, zz] [ -1, -1, 7, zz] PSLLDQ : (little-endian) left byte shift [ zz, 0, 1, 2, 3, 4, 5, 6] [ zz, zz, -1, -1, 2, 3, 4, -1] [ zz, zz, zz, zz, zz, zz, -1, 1] PSRLDQ : (little-endian) right byte shift [ 5, 6, 7, zz, zz, zz, zz, zz] [ -1, 5, 6, 7, zz, zz, zz, zz] [ 1, 2, -1, -1, -1, -1, zz, zz]
Definition at line 10921 of file X86ISelLowering.cpp.
References llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::MVT::i8, isSequentialOrUndefInRange(), llvm::Left, Size, llvm::ArrayRef< T >::size(), llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VSRLI.
Referenced by lowerVectorShuffleAsShift(), and matchUnaryPermuteVectorShuffle().
◆ matchVectorShuffleAsVPMOV()
Definition at line 10025 of file X86ISelLowering.cpp.
References isAnyInRange(), isSequentialOrUndefInRange(), Size, llvm::ArrayRef< T >::size(), llvm::ArrayRef< T >::slice(), and Split().
Referenced by lowerVectorShuffleWithVPMOV().
◆ matchVectorShuffleWithPACK()
|
static |
Definition at line 10120 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::ComputeNumSignBits(), createPackShuffleMask(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getHighBitsSet(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, isTargetShuffleEquivalent(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86ISD::PACKSS, and llvm::X86ISD::PACKUS.
Referenced by lowerVectorShuffleWithPACK(), and matchBinaryVectorShuffle().
◆ matchVectorShuffleWithSHUFPD()
|
static |
Definition at line 14572 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::SM_SentinelUndef, and std::swap().
Referenced by lowerVectorShuffleWithSHUFPD(), and matchBinaryPermuteVectorShuffle().
◆ matchVectorShuffleWithUNPCK()
|
static |
Definition at line 9913 of file X86ISelLowering.cpp.
References assert(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::createUnpackShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), isSequentialOrUndefOrZeroInRange(), isTargetShuffleEquivalent(), isUndefOrZero(), llvm::SM_SentinelUndef, std::swap(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, and llvm::MVT::v2i64.
Referenced by matchBinaryVectorShuffle().
◆ materializeVectorConstant()
|
static |
Create a vector constant without a load.
SSE/AVX provide the bare minimum functionality to do this, so it's all zeros, all ones, or some derivation that is cheap to calculate.
Definition at line 8519 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), getOnesVector(), llvm::SDValue::getSimpleValueType(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::v16i32, llvm::MVT::v4i32, and llvm::MVT::v8i32.
Referenced by LowerBUILD_VECTORAsVariablePermute().
◆ MayFoldIntoStore()
Definition at line 4510 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::hasOneUse(), llvm::ISD::isNormalStore(), and llvm::SDNode::use_begin().
Referenced by ExtractBitFromMaskVector().
◆ MayFoldIntoZeroExtend()
Definition at line 4514 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::hasOneUse(), llvm::SDNode::use_begin(), and llvm::ISD::ZERO_EXTEND.
Referenced by ExtractBitFromMaskVector().
◆ MayFoldLoad()
Definition at line 4506 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::hasOneUse(), and llvm::ISD::isNormalLoad().
Referenced by InsertBitToMaskVector(), llvm::X86TargetLowering::IsDesirableToPromoteOp(), isTruncWithZeroHighBitsInput(), and lowerVectorShuffleAsBroadcast().
◆ mayTailCallThisCC()
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 2926 of file X86ISelLowering.cpp.
References llvm::CallingConv::C, canGuaranteeTCO(), llvm::CallingConv::Win64, llvm::CallingConv::X86_64_SysV, llvm::CallingConv::X86_FastCall, llvm::CallingConv::X86_StdCall, llvm::CallingConv::X86_ThisCall, and llvm::CallingConv::X86_VectorCall.
Referenced by MatchingStackOffset(), and shouldGuaranteeTCO().
◆ needCarryOrOverflowFlag()
Definition at line 40201 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BRCOND, llvm::X86ISD::CMOV, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NO, llvm::X86::COND_O, llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDValue::getValueType(), llvm::MVT::i32, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
Referenced by combineCMP().
◆ negateFMAOpcode()
Definition at line 39551 of file X86ISelLowering.cpp.
References llvm::ISD::FMA, llvm::X86ISD::FMADD_RND, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUB_RND, llvm::X86ISD::FNMADD, llvm::X86ISD::FNMADD_RND, llvm::X86ISD::FNMSUB, llvm::X86ISD::FNMSUB_RND, and llvm_unreachable.
Referenced by combineFMA().
◆ onlyZeroFlagUsed()
Definition at line 40239 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BRCOND, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDValue::getValueType(), llvm::MVT::i32, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
Referenced by combineCMP().
◆ Passv64i1ArgInRegs()
|
static |
Breaks v64i1 value into two registers and adds the new node to the DAG.
Definition at line 2398 of file X86ISelLowering.cpp.
References llvm::CCValAssign::AExt, llvm::CCState::AnalyzeReturn(), llvm::ISD::ANY_EXTEND, assert(), llvm::CCValAssign::BCvt, llvm::EVT::bitsLT(), contains(), llvm::CCValAssign::convertToReg(), llvm::ISD::CopyToReg, llvm::MachineRegisterInfo::disableCalleeSavedRegister(), E, llvm::N86::EAX, llvm::SmallVectorBase::empty(), errorUnsupported(), llvm::ISD::EXTRACT_ELEMENT, llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::f80, llvm::ISD::FP_EXTEND, llvm::CCValAssign::FPExt, llvm::SelectionDAG::getBitcast(), llvm::X86MachineFunctionInfo::getBytesToPopOnReturn(), llvm::X86RegisterInfo::getCalleeSavedRegsViaCopy(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getDataLayout(), llvm::MachineFunction::getDataLayout(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getInfo(), llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocReg(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getNumValues(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::MachineFunction::getRegInfo(), llvm::SelectionDAG::getRegister(), llvm::X86Subtarget::getRegisterInfo(), llvm::TargetLoweringBase::getRegisterType(), llvm::X86MachineFunctionInfo::getSRetReturnReg(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::getTargetTriple(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::CCValAssign::getValVT(), llvm::EVT::getVectorElementType(), llvm::MVT::Glue, llvm::X86Subtarget::hasBWI(), llvm::Function::hasFnAttribute(), llvm::SDNode::hasNUsesOfValue(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MipsISD::Hi, I, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86ISD::IRET, llvm::X86Subtarget::is32Bit(), llvm::X86Subtarget::is64Bit(), llvm::Triple::isOSDarwin(), llvm::CCValAssign::isRegLoc(), llvm::X86TargetLowering::isScalarFPTypeInSSEReg(), llvm::X86Subtarget::isTarget64BitILP32(), llvm::EVT::isVector(), llvm_unreachable, llvm::MipsISD::Lo, lowerMasksToReg(), N, llvm::CCValAssign::needsCustom(), llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), Reg, llvm::report_fatal_error(), llvm::X86ISD::RET_FLAG, llvm::ISD::SCALAR_TO_VECTOR, llvm::CCValAssign::SExt, llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase::size(), TRI, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v64i1, llvm::CallingConv::X86_INTR, llvm::CallingConv::X86_RegCall, llvm::MVT::x86mmx, llvm::ISD::ZERO_EXTEND, and llvm::CCValAssign::ZExt.
Referenced by getMOVL().
◆ peekThroughEXTRACT_SUBVECTORs()
Definition at line 5690 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getOpcode(), and llvm::SDValue::getOperand().
Referenced by IsSplatVector().
◆ promoteExtBeforeAdd()
|
static |
sext(add_nsw(x, C)) –> add(sext(x), C_sext) zext(add_nuw(x, C)) –> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.
This can eliminate extend, add, and shift instructions.
Definition at line 39136 of file X86ISelLowering.cpp.
References llvm::MCID::Add, llvm::ISD::ADD, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNodeFlags::hasNoSignedWrap(), llvm::SDNodeFlags::hasNoUnsignedWrap(), llvm::MVT::i64, llvm::SDNodeFlags::setNoSignedWrap(), llvm::SDNodeFlags::setNoUnsignedWrap(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SDNode::uses(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
◆ PromoteMaskArithmetic()
|
static |
Definition at line 36034 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm::EVT::isVector(), llvm_unreachable, llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
◆ recoverFramePointer()
|
static |
When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.
Here's the math: RegNodeBase = EntryEBP - RegNodeSize ParentFP = RegNodeBase - ParentFrameOffset Subtracting RegNodeSize takes us to the offset of the registration node, and subtracting the offset (negative on x86) takes us back to the parent FP.
Definition at line 21487 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ADX, llvm::ISD::AND, assert(), llvm::CMP_MASK_CC, llvm::CMP_MASK_SCALAR_CC, llvm::COMI, llvm::COMI_RM, llvm::COMPRESS_EXPAND_IN_REG, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_O, llvm::X86::COND_P, llvm::X86::COND_S, llvm::X86::CUR_DIRECTION, llvm::CVTPD2I_MASK, llvm::CVTPD2PS, llvm::CVTPD2PS_MASK, llvm::CVTPD2PS_RND_MASK, llvm::CVTPS2PH_MASK, llvm::GlobalValue::dropLLVMManglingEscape(), llvm::dyn_cast(), llvm::Intrinsic::eh_recoverfp, llvm::Intrinsic::eh_sjlj_lsda, llvm::FIXUPIMM, llvm::FIXUPIMM_MASKZ, llvm::FIXUPIMMS, llvm::FIXUPIMMS_MASKZ, llvm::ISD::FP_ROUND, llvm::FPCLASSS, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_RND, llvm::X86RegisterInfo::getBaseRegister(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFunctionNumber(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::getIntrinsicWithoutChain(), llvm::SelectionDAG::getMachineFunction(), getMaskNode(), llvm::SelectionDAG::getMCSymbol(), llvm::SelectionDAG::getMergeValues(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::MCContext::getOrCreateLSDASymbol(), llvm::MCContext::getOrCreateParentFrameOffsetSymbol(), llvm::TargetLoweringBase::getPointerTy(), llvm::X86RegisterInfo::getPtrSizedFrameRegister(), llvm::X86Subtarget::getRegisterInfo(), getScalarMaskingNode(), getSEHRegistrationNodeSize(), getSETCC(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), getVectorMaskingNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86RegisterInfo::hasBasePointer(), llvm::Function::hasPersonalityFn(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::IFMA_OP, llvm::ISD::INSERT_SUBVECTOR, llvm::INTR_TYPE_1OP, llvm::INTR_TYPE_1OP_MASK, llvm::INTR_TYPE_1OP_MASK_RM, llvm::INTR_TYPE_2OP, llvm::INTR_TYPE_2OP_MASK, llvm::INTR_TYPE_2OP_MASK_RM, llvm::INTR_TYPE_3OP, llvm::INTR_TYPE_3OP_IMM8, llvm::INTR_TYPE_3OP_MASK, llvm::INTR_TYPE_3OP_SCALAR_MASK, llvm::INTR_TYPE_4OP, llvm::INTR_TYPE_SCALAR_MASK, llvm::INTR_TYPE_SCALAR_MASK_RM, llvm::X86Subtarget::is64Bit(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::X86ISD::KTEST, LLVM_FALLTHROUGH, llvm_unreachable, llvm::ISD::LOCAL_RECOVER, llvm::Intrinsic::localaddress, llvm::BitmaskEnumDetail::Mask(), llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::ISD::OR, llvm::X86ISD::PCMPESTR, llvm::X86ISD::PCMPISTR, llvm::X86ISD::PTEST, Reg, llvm::report_fatal_error(), Results, llvm::ROUNDP, llvm::ROUNDS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SUB, llvm::X86ISD::TESTP, llvm::ISD::TRUNCATE, llvm::TRUNCATE_TO_REG, llvm::IntrinsicData::Type, llvm::MVT::v16i1, llvm::MVT::v16i8, llvm::MVT::v1i1, llvm::MVT::v8i1, llvm::VPERM_2OP, llvm::X86ISD::VRNDSCALE, llvm::X86ISD::VRNDSCALES, llvm::VSHIFT, llvm::X86ISD::Wrapper, llvm::Intrinsic::x86_avx512_ktestc_b, llvm::Intrinsic::x86_avx512_ktestc_d, llvm::Intrinsic::x86_avx512_ktestc_q, llvm::Intrinsic::x86_avx512_ktestc_w, llvm::Intrinsic::x86_avx512_ktestz_b, llvm::Intrinsic::x86_avx512_ktestz_d, llvm::Intrinsic::x86_avx512_ktestz_q, llvm::Intrinsic::x86_avx512_ktestz_w, llvm::Intrinsic::x86_avx_ptestc_256, llvm::Intrinsic::x86_avx_ptestnzc_256, llvm::Intrinsic::x86_avx_ptestz_256, llvm::Intrinsic::x86_avx_vtestc_pd, llvm::Intrinsic::x86_avx_vtestc_pd_256, llvm::Intrinsic::x86_avx_vtestc_ps, llvm::Intrinsic::x86_avx_vtestc_ps_256, llvm::Intrinsic::x86_avx_vtestnzc_pd, llvm::Intrinsic::x86_avx_vtestnzc_pd_256, llvm::Intrinsic::x86_avx_vtestnzc_ps, llvm::Intrinsic::x86_avx_vtestnzc_ps_256, llvm::Intrinsic::x86_avx_vtestz_pd, llvm::Intrinsic::x86_avx_vtestz_pd_256, llvm::Intrinsic::x86_avx_vtestz_ps, llvm::Intrinsic::x86_avx_vtestz_ps_256, llvm::Intrinsic::x86_seh_lsda, llvm::Intrinsic::x86_sse41_ptestc, llvm::Intrinsic::x86_sse41_ptestnzc, llvm::Intrinsic::x86_sse41_ptestz, llvm::Intrinsic::x86_sse42_pcmpestri128, llvm::Intrinsic::x86_sse42_pcmpestria128, llvm::Intrinsic::x86_sse42_pcmpestric128, llvm::Intrinsic::x86_sse42_pcmpestrio128, llvm::Intrinsic::x86_sse42_pcmpestris128, llvm::Intrinsic::x86_sse42_pcmpestriz128, llvm::Intrinsic::x86_sse42_pcmpestrm128, llvm::Intrinsic::x86_sse42_pcmpistri128, llvm::Intrinsic::x86_sse42_pcmpistria128, llvm::Intrinsic::x86_sse42_pcmpistric128, llvm::Intrinsic::x86_sse42_pcmpistrio128, llvm::Intrinsic::x86_sse42_pcmpistris128, llvm::Intrinsic::x86_sse42_pcmpistriz128, llvm::Intrinsic::x86_sse42_pcmpistrm128, and llvm::ISD::ZERO_EXTEND.
◆ reduceMaskedLoadToScalarLoad()
|
static |
If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.
Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.
Definition at line 37359 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MaskedLoadSDNode::getPassThru(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_VECTOR_ELT, and llvm::SPII::Load.
Referenced by combineMaskedLoad().
◆ reduceMaskedStoreToScalarStore()
|
static |
If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store.
Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.
Definition at line 37526 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getStore(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), and llvm::EVT::getVectorElementType().
Referenced by combineMaskedStore().
◆ reduceVMULWidth()
|
static |
When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.
Two typical patterns are handled: Pattern1: %2 = sext/zext <N x="" i8>=""> %1 to <N x="" i32>=""> %4 = sext/zext <N x="" i8>=""> %3 to <N x="" i32>=""> %5 = mul <N x="" i32>=""> %2, %4
Pattern2: %2 = zext/sext <N x="" i16>=""> %1 to <N x="" i32>=""> %4 = zext/sext <N x="" i16>=""> %3 to <N x="" i32>=""> or %4 = build_vector <N x="" i32>=""> C1, ..., CN (C1..CN are constants) %5 = mul <N x="" i32>=""> %2, %4
There are four mul shrinking modes: If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is -128 to 128, and the scalar value range of %4 is also -128 to 128, generate pmullw+sext32 for it (MULS8 mode). If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is 0 to 255, and the scalar value range of %4 is also 0 to 255, generate pmullw+zext32 for it (MULU8 mode). If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767, generate pmullw+pmulhw for it (MULS16 mode). If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is 0 to 65535, and the scalar value range of %4 is also 0 to 65535, generate pmullw+pmulhuw for it (MULU16 mode).
Definition at line 35047 of file X86ISelLowering.cpp.
References canReduceVMulWidth(), llvm::ISD::CONCAT_VECTORS, ExperimentalVectorWideningLegalization, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getUnpackl(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::X86Subtarget::isPMULLDSlow(), Mode, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, MULS16, MULS8, MULU8, llvm::Function::optForMinSize(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::TRUNCATE, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by combineMul().
◆ resolveTargetShuffleInputs()
|
static |
Calls setTargetShuffleZeroElements to resolve a target shuffle mask's inputs and set the SM_SentinelUndef and SM_SentinelZero values.
Then check the remaining input indices in case we now have a unary shuffle and adjust the inputs accordingly. Returns true if the target shuffle mask was decoded.
Definition at line 6780 of file X86ISelLowering.cpp.
References getFauxShuffleMask(), resolveTargetShuffleInputsAndMask(), and setTargetShuffleZeroElements().
Referenced by combineExtractWithShuffle(), combineX86ShufflesRecursively(), getFauxShuffleMask(), and setTargetShuffleZeroElements().
◆ resolveTargetShuffleInputsAndMask()
|
static |
Removes unused shuffle source inputs and adjusts the shuffle mask accordingly.
Definition at line 6749 of file X86ISelLowering.cpp.
References llvm::any_of(), isUndef(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorBase::size(), and llvm::SM_SentinelUndef.
Referenced by combineX86ShufflesRecursively(), and resolveTargetShuffleInputs().
◆ scaleVectorShuffleBlendMask()
|
static |
Definition at line 10293 of file X86ISelLowering.cpp.
References Size.
Referenced by lowerVectorShuffleAsBlend(), and matchBinaryPermuteVectorShuffle().
◆ SegmentOffset()
Definition at line 2249 of file X86ISelLowering.cpp.
References llvm::ConstantInt::get(), llvm::IRBuilderBase::getContext(), llvm::Type::getInt32Ty(), llvm::Type::getInt8PtrTy(), and llvm::ConstantExpr::getIntToPtr().
Referenced by llvm::object::MachORebaseEntry::address(), llvm::object::MachOBindEntry::address(), llvm::X86TargetLowering::getIRStackGuard(), llvm::X86TargetLowering::getSafeStackPointerLocation(), llvm::object::MachORebaseEntry::moveNext(), llvm::object::MachOBindEntry::moveNext(), llvm::object::MachORebaseEntry::sectionName(), llvm::object::MachOBindEntry::sectionName(), llvm::object::MachORebaseEntry::segmentOffset(), and llvm::object::MachOBindEntry::segmentOffset().
◆ setTargetShuffleZeroElements()
|
static |
Check a target shuffle mask's inputs to see if we can set any values to SM_SentinelZero - this is for elements that are known to be zero (not just zeroable) from their inputs.
Returns true if the target shuffle mask was decoded.
Definition at line 6390 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetShuffleMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::isFloatingPoint(), isTargetShuffle(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::BitmaskEnumDetail::Mask(), llvm::peekThroughBitcasts(), resolveTargetShuffleInputs(), llvm::ISD::SCALAR_TO_VECTOR, Size, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by combineTargetShuffle(), and resolveTargetShuffleInputs().
◆ shouldGuaranteeTCO()
|
static |
Return true if the function is being made into a tailcall target by changing its ABI.
Definition at line 2945 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, canGuaranteeTCO(), llvm::MachineFrameInfo::CreateFixedObject(), llvm::ISD::ArgFlagsTy::getByValSize(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getCallingConv(), llvm::SelectionDAG::getDataLayout(), llvm::MachinePointerInfo::getFixedStack(), llvm::Function::getFnAttribute(), llvm::SelectionDAG::getFrameIndex(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocMemOffset(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::MachineFrameInfo::getObjectIndexBegin(), llvm::MachineFrameInfo::getObjectOffset(), llvm::MachineFrameInfo::getObjectSize(), llvm::Instruction::getParent(), llvm::BasicBlock::getParent(), llvm::TargetLoweringBase::getPointerTy(), llvm::MVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::EVT::getStoreSize(), llvm::SelectionDAG::getTarget(), llvm::CCValAssign::getValVT(), llvm::TargetOptions::GuaranteedTailCallOpt, llvm::MVT::i1, llvm::CCValAssign::Indirect, llvm::X86Subtarget::is64Bit(), llvm::ISD::ArgFlagsTy::isByVal(), llvm::ISD::ArgFlagsTy::isCopyElisionCandidate(), llvm::CCValAssign::isExtInLoc(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::CallInst::isTailCall(), llvm::MVT::isVector(), mayTailCallThisCC(), llvm::TargetMachine::Options, llvm::ISD::SCALAR_TO_VECTOR, llvm::MachineFrameInfo::setObjectOffset(), llvm::MachineFrameInfo::setObjectSExt(), llvm::MachineFrameInfo::setObjectZExt(), llvm::CCValAssign::SExt, llvm::SmallVectorBase::size(), llvm::ISD::TRUNCATE, llvm::CallingConv::X86_INTR, and llvm::CCValAssign::ZExt.
Referenced by llvm::X86::isCalleePop(), and isSortedByValueNo().
◆ shouldUseHorizontalOp()
|
static |
Horizontal vector math instructions may be slower than normal math with shuffles.
Limit horizontal op codegen based on size/speed trade-offs, uarch implementation, and likely shuffle complexity of the alternate sequence.
Definition at line 18501 of file X86ISelLowering.cpp.
References llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::X86Subtarget::hasFastHorizontalOps(), and llvm::Function::optForSize().
Referenced by combineAdd(), combineFaddFsub(), combineSub(), and lowerAddSubToHorizontalOp().
◆ split256IntArith()
|
static |
Break a 256-bit integer operation into two new 128-bit ones and then concatenate the result back.
Definition at line 23479 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), and llvm::MVT::isInteger().
Referenced by combineExtractSubvector(), LowerADDSAT_SUBSAT(), lowerAddSub(), LowerMINMAX(), LowerMUL(), LowerMULH(), LowerRotate(), and LowerShift().
◆ split512IntArith()
|
static |
Break a 512-bit integer operation into two new 256-bit ones and then concatenate the result back.
Definition at line 23508 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract256BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is512BitVector(), and llvm::MVT::isInteger().
Referenced by LowerMULH().
◆ SplitAndExtendv16i1()
|
static |
Definition at line 18024 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MipsISD::Hi, llvm::MipsISD::Lo, llvm::ISD::TRUNCATE, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v8i1, and llvm::MVT::v8i16.
Referenced by LowerSIGN_EXTEND_Mask(), and LowerZERO_EXTEND_Mask().
◆ splitAndLowerVectorShuffle()
|
static |
Generic routine to split vector shuffle into half-sized shuffles.
This routine just extracts two subvectors, shuffles them independently, and then concatenates them back together. This should work effectively with all AVX vector shuffle types.
Definition at line 13693 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MipsISD::Hi, llvm::MipsISD::Lo, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ArrayRef< T >::size(), and llvm::ArrayRef< T >::slice().
Referenced by lower256BitVectorShuffle(), lowerV64I8VectorShuffle(), lowerVectorShuffleAsLanePermuteAndBlend(), and lowerVectorShuffleAsSplitOrBlend().
◆ SplitOpsAndApply()
| SDValue SplitOpsAndApply | ( | SelectionDAG & | DAG, |
| const X86Subtarget & | Subtarget, | ||
| const SDLoc & | DL, | ||
| EVT | VT, | ||
| ArrayRef< SDValue > | Ops, | ||
| F | Builder, | ||
| bool | CheckBWI = true |
||
| ) |
Definition at line 5420 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::X86Subtarget::useAVX512Regs(), and llvm::X86Subtarget::useBWIRegs().
Referenced by combineAdd(), combineLoopMAddPattern(), combineMulToPMADDWD(), combineMulToPMULDQ(), combineSub(), combineSubToSubus(), createPSADBW(), createVariablePermute(), detectAVGPattern(), detectPMADDUBSW(), matchPMADDWD(), and matchPMADDWD_2().
◆ STATISTIC()
| STATISTIC | ( | NumTailCalls | , |
| "Number of tail calls" | |||
| ) |
◆ SupportedVectorShiftWithBaseAmnt()
|
static |
Definition at line 24092 of file X86ISelLowering.cpp.
References SupportedVectorShiftWithImm().
Referenced by LowerScalarVariableShift().
◆ SupportedVectorShiftWithImm()
|
static |
Definition at line 24072 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::SRA, llvm::MVT::v2i64, and llvm::MVT::v4i64.
Referenced by combineAndMaskToShift(), LowerScalarImmediateShift(), and SupportedVectorShiftWithBaseAmnt().
◆ SupportedVectorVarShift()
|
static |
Definition at line 24099 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::ISD::SRA, llvm::MVT::v2i64, and llvm::MVT::v4i64.
Referenced by LowerRotate(), and LowerShift().
◆ TranslateIntegerX86CC()
|
static |
Definition at line 4668 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, and llvm::ISD::SETULT.
Referenced by EmitKORTEST(), and TranslateX86CC().
◆ TranslateX86CC()
|
static |
Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make.
Definition at line 4687 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_P, llvm::X86::COND_S, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), llvm::ISD::isNON_EXTLoad(), llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, std::swap(), and TranslateIntegerX86CC().
Referenced by EmitKORTEST().
◆ translateX86FSETCC()
|
static |
Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs.
Definition at line 19200 of file X86ISelLowering.cpp.
References LLVM_FALLTHROUGH, llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, and std::swap().
Referenced by isTruncWithZeroHighBitsInput(), and LowerVSETCC().
◆ truncateVectorWithPACK()
|
static |
Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
It makes use of the fact that vectors with enough leading sign/zero bits prevent the PACKSS/PACKUS from saturating the results. AVX2 (Int256) sub-targets require extra shuffling as the PACK*S operates within each 128-bit lane.
Definition at line 18112 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::tgtok::In, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::MipsISD::Lo, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, and llvm::MVT::v4i64.
Referenced by combineTruncateWithSat(), combineVectorSignBitsTruncation(), combineVectorTruncationWithPACKSS(), combineVectorTruncationWithPACKUS(), and LowerTruncateVecI1().
◆ widenSubVector()
|
static |
Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
Definition at line 5400 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getZeroVector(), and llvm::ISD::INSERT_SUBVECTOR.
Referenced by createVariablePermute(), LowerCONCAT_VECTORSvXi1(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
◆ XFormVExtractWithShuffleIntoLoad()
|
static |
Check if a vector extract from a target-specific shuffle of a load can be folded into a single element load.
Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but shuffles have been custom lowered so we need to handle those here.
Definition at line 32783 of file X86ISelLowering.cpp.
References llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, llvm::any_of(), assert(), llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DataLayout::getABITypeAlignment(), llvm::MemSDNode::getAlignment(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), isTargetShuffle(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::ISD::LOAD, llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorBase::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineExtractVectorElt().
Variable Documentation
◆ ExperimentalPrefLoopAlignment
|
static |
Referenced by llvm::X86TargetLowering::X86TargetLowering().
◆ ExperimentalVectorWideningLegalization
|
static |
Referenced by combineMulToPMADDWD(), combinePMULH(), combineSelect(), combineSetCC(), combineToExtendVectorInReg(), llvm::X86TargetLowering::getPreferredVectorAction(), LowerAVXExtend(), LowerSIGN_EXTEND(), reduceVMULWidth(), llvm::X86TargetLowering::ReplaceNodeResults(), and llvm::X86TargetLowering::X86TargetLowering().
◆ MulConstantOptimization
|
static |
Referenced by combineMul().
