X86ISelDAGToDAG.cpp File Reference

#include "X86.h"
#include "X86MachineFunctionInfo.h"
#include "X86RegisterInfo.h"
#include "X86Subtarget.h"
#include "X86TargetMachine.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/SelectionDAGISel.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <stdint.h>
#include "X86GenDAGISel.inc"

Include dependency graph for X86ISelDAGToDAG.cpp:

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "x86-isel"

Functions
	STATISTIC (NumLoadMoved, "Number of loads moved below TokenFactor")
static bool	isLegalMaskCompare (SDNode *N, const X86Subtarget *Subtarget)
static void	moveBelowOrigChain (SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
	Replace the original chain operand of the call with load's chain operand and move load below the call's chain operand. More...
static bool	isCalleeLoad (SDValue Callee, SDValue &Chain, bool HasCallSeq)
	Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading up to the call. More...
static bool	isDispSafeForFrameIndex (int64_t Val)
static void	insertDAGNode (SelectionDAG &DAG, SDValue Pos, SDValue N)
static bool	foldMaskAndShiftToExtract (SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool	foldMaskedShiftToScaledMask (SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool	foldMaskAndShiftToScale (SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
static bool	foldMaskedShiftToBEXTR (SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)
static bool	hasSingleUsesFromRoot (SDNode *Root, SDNode *User)
static X86::CondCode	getCondFromOpc (unsigned Opc)
static bool	mayUseCarryFlag (X86::CondCode CC)
static bool	isFusableLoadOpStorePattern (StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, unsigned LoadOpNo, LoadSDNode *&LoadNode, SDValue &InputChain)
	Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to modify transformation. More...

Variables
static cl::opt< bool >	AndImmShrink ("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "x86-isel"

Definition at line 40 of file X86ISelDAGToDAG.cpp.

Function Documentation

foldMaskAndShiftToExtract()

static bool foldMaskAndShiftToExtract ( SelectionDAG &  DAG, SDValue  N, uint64_t  Mask, SDValue  Shift, SDValue  X, X86ISelAddressMode &  AM  )

static

Definition at line 1293 of file X86ISelDAGToDAG.cpp.

References llvm::ISD::AND, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i8, insertDAGNode(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SHL, and llvm::ISD::SRL.

Referenced by foldMaskedShiftToBEXTR().

foldMaskAndShiftToScale()

static bool foldMaskAndShiftToScale ( SelectionDAG &  DAG, SDValue  N, uint64_t  Mask, SDValue  Shift, SDValue  X, X86ISelAddressMode &  AM  )

static

Definition at line 1403 of file X86ISelDAGToDAG.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::countLeadingZeros(), llvm::countTrailingOnes(), llvm::countTrailingZeros(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i8, insertDAGNode(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SHL, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.

Referenced by foldMaskedShiftToBEXTR().

foldMaskedShiftToBEXTR()

static bool foldMaskedShiftToBEXTR ( SelectionDAG &  DAG, SDValue  N, uint64_t  Mask, SDValue  Shift, SDValue  X, X86ISelAddressMode &  AM, const X86Subtarget &  Subtarget  )

static

Definition at line 1489 of file X86ISelDAGToDAG.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::Constant, llvm::ISD::CopyFromReg, llvm::countTrailingZeros(), llvm::dbgs(), llvm::ISD::DELETED_NODE, llvm::X86ISD::EH_SJLJ_LONGJMP, llvm::X86ISD::EH_SJLJ_SETJMP, foldMaskAndShiftToExtract(), foldMaskAndShiftToScale(), foldMaskedShiftToScaledMask(), llvm::ISD::FrameIndex, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::ConstantSDNode::getSExtValue(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::HandleSDNode::getValue(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasBMI(), llvm::X86Subtarget::hasFastBEXTR(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasTBM(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, insertDAGNode(), llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, isDispSafeForFrameIndex(), llvm::isShiftedMask_64(), LLVM_DEBUG, LLVM_FALLTHROUGH, llvm::ISD::LOAD, llvm::ISD::LOCAL_RECOVER, llvm::BitmaskEnumDetail::Mask(), llvm::ISD::MUL, llvm::X86ISD::MUL_IMM, N, llvm::ISD::OR, Reg, llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SHL, llvm::ISD::SMUL_LOHI, llvm::ISD::SRL, llvm::ISD::SUB, llvm::X86ISD::TLSCALL, llvm::ISD::TRUNCATE, llvm::ISD::UMUL_LOHI, llvm::X86ISD::Wrapper, llvm::X86ISD::WrapperRIP, X, and llvm::ISD::ZERO_EXTEND.

foldMaskedShiftToScaledMask()

static bool foldMaskedShiftToScaledMask ( SelectionDAG &  DAG, SDValue  N, uint64_t  Mask, SDValue  Shift, SDValue  X, X86ISelAddressMode &  AM  )

static

Definition at line 1336 of file X86ISelDAGToDAG.cpp.

References llvm::ISD::AND, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::hasOneUse(), insertDAGNode(), llvm::SelectionDAG::ReplaceAllUsesWith(), and llvm::ISD::SHL.

Referenced by foldMaskedShiftToBEXTR().

getCondFromOpc()

static X86::CondCode getCondFromOpc ( unsigned  Opc )

static

Definition at line 2285 of file X86ISelDAGToDAG.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_NE, llvm::X86::COND_NO, llvm::X86::COND_NP, llvm::X86::COND_O, llvm::X86::COND_P, llvm::ISD::CopyToReg, llvm::X86::getCondFromBranchOpc(), llvm::X86::getCondFromCMovOpc(), llvm::X86::getCondFromSETOpc(), getReg(), llvm::SDValue::getResNo(), llvm::SDNode::use_begin(), and llvm::SDNode::use_end().

Referenced by mayUseCarryFlag().

hasSingleUsesFromRoot()

static bool hasSingleUsesFromRoot ( SDNode *  Root, SDNode *  User  )

static

Definition at line 1960 of file X86ISelDAGToDAG.cpp.

References assert(), llvm::dyn_cast(), llvm::N86::EBX, llvm::LoadSDNode::getBasePtr(), llvm::MachineFunction::getDataLayout(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::RegisterSDNode::getReg(), llvm::ConstantRange::getSignedMax(), llvm::ConstantRange::getSignedMin(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::ConstantRange::getUnsignedMax(), llvm::SDValue::getValueType(), llvm::PPCISD::GlobalBaseReg, llvm::SDNode::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::isNON_EXTLoad(), llvm::isUInt< 32 >(), llvm::ARM_MB::LD, MI, N, P, llvm::NVPTX::PTXCvtMode::RN, llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::sge(), llvm::APInt::slt(), llvm::CodeModel::Small, T, llvm::ISD::TargetGlobalAddress, llvm::ISD::TargetGlobalTLSAddress, llvm::SystemZISD::TM, llvm::ISD::TRUNCATE, llvm::APInt::ult(), llvm::SDNode::use_begin(), llvm::X86ISD::VZEXT_LOAD, llvm::X86ISD::VZEXT_MOVL, and llvm::X86ISD::Wrapper.

insertDAGNode()

static void insertDAGNode ( SelectionDAG &  DAG, SDValue  Pos, SDValue  N  )

static

Definition at line 1275 of file X86ISelDAGToDAG.cpp.

References llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::SDValue::getNode(), llvm::SDNode::getNodeId(), llvm::SelectionDAGISel::getUninvalidatedNodeId(), llvm::SelectionDAGISel::InvalidateNodeId(), llvm::SelectionDAG::RepositionNode(), and llvm::SDNode::setNodeId().

Referenced by foldMaskAndShiftToExtract(), foldMaskAndShiftToScale(), foldMaskedShiftToBEXTR(), and foldMaskedShiftToScaledMask().

isCalleeLoad()

static bool isCalleeLoad ( SDValue  Callee, SDValue &  Chain, bool  HasCallSeq  )

static

Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading up to the call.

Return the CALLSEQ_START by reference as a second output. In the case of a tail call, there isn't a callseq node between the call chain and the load.

Definition at line 685 of file X86ISelDAGToDAG.cpp.

References llvm::X86ISD::AND, llvm::ISD::AND, llvm::AMDGPU::HSAMD::Kernel::Key::Args, assert(), llvm::CallingConv::C, llvm::X86ISD::CALL, llvm::ISD::CALLSEQ_START, llvm::dyn_cast(), E, llvm::X86II::EncodingMask, llvm::X86II::EVEX, llvm::ISD::EXTLOAD, F(), llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::LSBaseSDNode::getAddressingMode(), llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), llvm::LoadSDNode::getExtensionType(), llvm::MachineFunction::getFunction(), llvm::SDValue::getMachineOpcode(), llvm::SDNode::getMachineOpcode(), llvm::Value::getName(), llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::Type::getVoidTy(), llvm::SDNode::hasAnyUseOfValue(), llvm::GlobalValue::hasExternalLinkage(), llvm::SDValue::hasOneUse(), I, llvm::MVT::i32, llvm::MVT::i64, llvm::tgtok::In, llvm::SDValue::isMachineOpcode(), llvm::SDNode::isMachineOpcode(), llvm::SDNode::isOnlyUserOf(), llvm::SDValue::isOperandOf(), llvm::X86TargetLowering::isScalarFPTypeInSSEReg(), llvm::MVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::ARM_MB::LD, llvm_unreachable, llvm::SPII::Load, llvm::TargetLowering::LowerCallTo(), moveBelowOrigChain(), llvm::ISD::NON_EXTLOAD, llvm::CodeGenOpt::None, llvm::Function::optForMinSize(), llvm::Function::optForSize(), llvm::MVT::Other, llvm::TargetLowering::CallLoweringInfo::setChain(), llvm::SPII::Store, llvm::X86ISD::TC_RETURN, llvm::SystemZISD::TM, llvm::ISD::TokenFactor, llvm::ISD::UNINDEXED, llvm::SDNode::use_empty(), llvm::X86II::VEX, and llvm::X86II::XOP.

isDispSafeForFrameIndex()

static bool isDispSafeForFrameIndex ( int64_t  Val )

static

Definition at line 1067 of file X86ISelDAGToDAG.cpp.

References llvm::Address, C, llvm::Depth, G, llvm::MachinePointerInfo::getAddrSpace(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::HandleSDNode::getValue(), llvm::MVT::i16, llvm::MVT::i64, llvm::X86::isOffsetSuitableForCodeModel(), llvm::CodeModel::Large, llvm_unreachable, llvm::CodeModel::Medium, llvm::X86II::MO_NO_FLAG, llvm::CodeModel::Small, llvm::ISD::TargetGlobalTLSAddress, llvm::SystemZISD::TM, and llvm::X86ISD::WrapperRIP.

Referenced by foldMaskedShiftToBEXTR().

isFusableLoadOpStorePattern()

static bool isFusableLoadOpStorePattern ( StoreSDNode *  StoreNode, SDValue  StoredVal, SelectionDAG *  CurDAG, unsigned  LoadOpNo, LoadSDNode *&  LoadNode, SDValue &  InputChain  )

static

Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to modify transformation.

Definition at line 2447 of file X86ISelDAGToDAG.cpp.

References llvm::LoadSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::LoadSDNode::getOffset(), llvm::StoreSDNode::getOffset(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValue(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasPredecessorHelper(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::ISD::isNormalStore(), llvm::SPII::Load, llvm::SDNode::ops(), llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), and llvm::ISD::TokenFactor.

isLegalMaskCompare()

static bool isLegalMaskCompare ( SDNode *  N, const X86Subtarget *  Subtarget  )

static

Definition at line 485 of file X86ISelDAGToDAG.cpp.

References llvm::X86ISD::ADC, llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ISD::ADDCARRY, llvm::X86ISD::AND, llvm::ISD::AND, C, llvm::X86ISD::CMPM, llvm::X86ISD::CMPM_RND, llvm::dyn_cast(), llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_RND, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasVLX(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::SDValue::isUndef(), llvm::ISD::LOAD, llvm::CodeGenOpt::None, llvm::X86ISD::OR, llvm::ISD::OR, llvm::ISD::ROTL, llvm::X86ISD::SBB, llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::X86ISD::SUB, llvm::ISD::TargetGlobalTLSAddress, llvm::X86ISD::VFPCLASS, llvm::X86ISD::VFPCLASSS, llvm::X86ISD::Wrapper, llvm::X86ISD::XOR, and llvm::ISD::XOR.

mayUseCarryFlag()

static bool mayUseCarryFlag ( X86::CondCode  CC )

static

Definition at line 2373 of file X86ISelDAGToDAG.cpp.

References llvm::X86ISD::BRCOND, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NO, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_O, llvm::X86::COND_P, llvm::X86::COND_S, llvm::ISD::CopyToReg, getCondFromOpc(), getReg(), llvm::SDValue::getResNo(), llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().

moveBelowOrigChain()

static void moveBelowOrigChain ( SelectionDAG *  CurDAG, SDValue  Load, SDValue  Call, SDValue  OrigChain  )

static

Replace the original chain operand of the call with load's chain operand and move load below the call's chain operand.

Definition at line 650 of file X86ISelDAGToDAG.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::clear(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::MVT::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::ISD::TokenFactor, and llvm::SelectionDAG::UpdateNodeOperands().

Referenced by isCalleeLoad().

STATISTIC()

STATISTIC	(	NumLoadMoved	,
		"Number of loads moved below TokenFactor"
	)

Variable Documentation

AndImmShrink

cl::opt<bool> AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)

static