LoopVectorize.cpp File Reference

#include "llvm/Transforms/Vectorize/LoopVectorize.h"
#include "LoopVectorizationPlanner.h"
#include "VPRecipeBuilder.h"
#include "VPlanHCFGBuilder.h"
#include "VPlanHCFGTransforms.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseMapInfo.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpander.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/IR/Verifier.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopVersioning.h"
#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <functional>
#include <iterator>
#include <limits>
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>

Go to the source code of this file.

Classes
class	llvm::InnerLoopVectorizer
	InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization factor (VF). More...
class	llvm::InnerLoopUnroller
class	llvm::LoopVectorizationCostModel
	LoopVectorizationCostModel - estimates the expected speedups due to vectorization. More...
struct	llvm::LoopVectorizationCostModel::RegisterUsage
	A struct that represents some properties of the register usage of a loop. More...

Namespaces
	llvm
	This class represents lattice values for constants.

Macros
#define	LV_NAME   "loop-vectorize"
#define	DEBUG_TYPE   LV_NAME

Functions
	STATISTIC (LoopsVectorized, "Number of loops vectorized")
	STATISTIC (LoopsAnalyzed, "Number of loops analyzed for vectorization")
static Type *	ToVectorTy (Type *Scalar, unsigned VF)
	A helper function for converting Scalar types to vector types. More...
static Type *	getMemInstValueType (Value *I)
	A helper function that returns the type of loaded or stored value. More...
static bool	hasIrregularType (Type *Ty, const DataLayout &DL, unsigned VF)
	A helper function that returns true if the given type is irregular. More...
static unsigned	getReciprocalPredBlockProb ()
	A helper function that returns the reciprocal of the block probability of predicated blocks. More...
static Value *	addFastMathFlag (Value *V)
	A helper function that adds a 'fast' flag to floating-point operations. More...
static Constant *	getSignedIntOrFpConstant (Type *Ty, int64_t C)
	A helper function that returns an integer or floating-point constant with value C. More...
static Instruction *	getDebugLocFromInstOrOperands (Instruction *I)
	Look for a meaningful debug location on the instruction or it's operands. More...
static std::string	getDebugLocString (const Loop *L)
static bool	isExplicitVecOuterLoop (Loop *OuterLp, OptimizationRemarkEmitter *ORE)
static void	collectSupportedLoops (Loop &L, LoopInfo *LI, OptimizationRemarkEmitter *ORE, SmallVectorImpl< Loop *> &V)
static bool	useMaskedInterleavedAccesses (const TargetTransformInfo &TTI)
static void	cse (BasicBlock *BB)
	Perform cse of induction variable instructions. More...
static unsigned	getScalarizationOverhead (Instruction *I, unsigned VF, const TargetTransformInfo &TTI)
	Estimate the overhead of scalarizing an instruction. More...
static unsigned	getVectorCallCost (CallInst *CI, unsigned VF, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI, bool &NeedToScalarize)
static unsigned	getVectorIntrinsicCost (CallInst *CI, unsigned VF, const TargetTransformInfo &TTI, const TargetLibraryInfo *TLI)
static Type *	smallestIntegerVectorType (Type *T1, Type *T2)
static Type *	largestIntegerVectorType (Type *T1, Type *T2)
static bool	mayDivideByZero (Instruction &I)
	A helper function for checking whether an integer division-related instruction may divide by zero (in which case it must be predicated if executed conditionally in the scalar code). More...
static const SCEV *	getAddressAccessSCEV (Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)
	Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence. More...
static bool	isStrideMul (Instruction *I, LoopVectorizationLegality *Legal)
Pass *	llvm::createLoopVectorizePass (bool InterleaveOnlyWhenForced=false, bool VectorizeOnlyWhenForced=false)
static void	AddRuntimeUnrollDisableMetaData (Loop *L)
static bool	processLoopInVPlanNativePath (Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT, LoopVectorizationLegality *LVL, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC, OptimizationRemarkEmitter *ORE, LoopVectorizeHints &Hints)

Variables
static cl::opt< unsigned >	TinyTripCountVectorThreshold ("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))
	Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred. More...
static cl::opt< bool >	MaximizeBandwidth ("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))
static cl::opt< bool >	EnableInterleavedMemAccesses ("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))
static cl::opt< bool >	EnableMaskedInterleavedMemAccesses ("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))
	An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps. More...
static const unsigned	TinyTripCountInterleaveThreshold = 128
	We don't interleave loops with a known constant trip count below this number. More...
static cl::opt< unsigned >	ForceTargetNumScalarRegs ("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))
static cl::opt< unsigned >	ForceTargetNumVectorRegs ("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))
static cl::opt< unsigned >	ForceTargetMaxScalarInterleaveFactor ("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))
static cl::opt< unsigned >	ForceTargetMaxVectorInterleaveFactor ("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))
static cl::opt< unsigned >	ForceTargetInstructionCost ("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))
static cl::opt< unsigned >	SmallLoopCost ("small-loop-cost", cl::init(20), cl::Hidden, cl::desc("The cost of a loop that is considered 'small' by the interleaver."))
static cl::opt< bool >	LoopVectorizeWithBlockFrequency ("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))
static cl::opt< bool >	EnableLoadStoreRuntimeInterleave ("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc("Enable runtime interleaving until load/store ports are saturated"))
static cl::opt< unsigned >	NumberOfStoresToPredicate ("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))
	The number of stores in a loop that are allowed to need predication. More...
static cl::opt< bool >	EnableIndVarRegisterHeur ("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))
static cl::opt< bool >	EnableCondStoresVectorization ("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))
static cl::opt< unsigned >	MaxNestedScalarReductionIC ("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))
cl::opt< bool >	EnableVPlanNativePath ("enable-vplan-native-path", cl::init(false), cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
static cl::opt< bool >	VPlanBuildStressTest ("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc("Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))
static const char	lv_name [] = "Loop Vectorization"
static const char *const	LLVMLoopVectorizeFollowupAll
static const char *const	LLVMLoopVectorizeFollowupVectorized
static const char *const	LLVMLoopVectorizeFollowupEpilogue

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   LV_NAME

Definition at line 153 of file LoopVectorize.cpp.

LV_NAME

#define LV_NAME   "loop-vectorize"

Definition at line 152 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizePass::processLoop().

Function Documentation

addFastMathFlag()

static Value* addFastMathFlag ( Value *  V )

static

A helper function that adds a 'fast' flag to floating-point operations.

Definition at line 313 of file LoopVectorize.cpp.

References llvm::FastMathFlags::setFast().

Referenced by llvm::InnerLoopVectorizer::buildScalarSteps(), llvm::LoopVectorizationPlanner::collectTriviallyDeadInstructions(), and llvm::InnerLoopVectorizer::fixReduction().

AddRuntimeUnrollDisableMetaData()

static void AddRuntimeUnrollDisableMetaData ( Loop *  L )

static

Definition at line 6257 of file LoopVectorize.cpp.

References Context, llvm::dyn_cast(), llvm::MDString::get(), llvm::MDNode::get(), llvm::BasicBlock::getContext(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::Loop::getLoopID(), llvm::MDNode::getNumOperands(), llvm::MDNode::getOperand(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::MDNode::replaceOperandWith(), and llvm::Loop::setLoopID().

Referenced by llvm::LoopVectorizePass::processLoop().

collectSupportedLoops()

static void collectSupportedLoops ( Loop &  L, LoopInfo *  LI, OptimizationRemarkEmitter *  ORE, SmallVectorImpl< Loop *> &  V  )

static

Definition at line 1392 of file LoopVectorize.cpp.

References llvm::InnerLoopVectorizer::AA, llvm::InnerLoopVectorizer::AC, llvm::AnalysisUsage::addPreserved(), llvm::AnalysisUsage::addRequired(), llvm::AMDGPUISD::BFI, llvm::InnerLoopVectorizer::DT, llvm::LoopBase< BlockT, LoopT >::empty(), EnableVPlanNativePath, F(), llvm::PassRegistry::getPassRegistry(), llvm::initializeLoopVectorizePass(), llvm::LoopVectorizePass::InterleaveOnlyWhenForced, isExplicitVecOuterLoop(), llvm::InnerLoopVectorizer::LI, llvm::InnerLoopVectorizer::ORE, llvm::LoopBlocksRPO::perform(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::LoopVectorizePass::runImpl(), runOnFunction(), llvm::InnerLoopVectorizer::TLI, llvm::InnerLoopVectorizer::TTI, llvm::LoopVectorizePass::VectorizeOnlyWhenForced, and VPlanBuildStressTest.

Referenced by llvm::LoopVectorizePass::runImpl().

cse()

static void cse ( BasicBlock *  BB )

static

Perform cse of induction variable instructions.

Definition at line 3004 of file LoopVectorize.cpp.

References llvm::BasicBlock::begin(), E, llvm::BasicBlock::end(), llvm::Instruction::eraseFromParent(), I, llvm::tgtok::In, llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), and llvm::Value::replaceAllUsesWith().

Referenced by llvm::InnerLoopVectorizer::fixVectorizedLoop().

getAddressAccessSCEV()

static const SCEV* getAddressAccessSCEV ( Value *  Ptr, LoopVectorizationLegality *  Legal, PredicatedScalarEvolution &  PSE, const Loop *  TheLoop  )

static

Gets Address Access SCEV after verifying that the access pattern is loop invariant except the induction variable dependence.

This SCEV can be sent to the Target in order to estimate the address calculation cost.

Definition at line 5431 of file LoopVectorize.cpp.

References llvm::dyn_cast(), llvm::PredicatedScalarEvolution::getSCEV(), llvm::PredicatedScalarEvolution::getSE(), and llvm::LoopVectorizationLegality::isInductionVariable().

Referenced by isStrideMul().

getDebugLocFromInstOrOperands()

static Instruction* getDebugLocFromInstOrOperands ( Instruction *  I )

static

Look for a meaningful debug location on the instruction or it's operands.

Definition at line 741 of file LoopVectorize.cpp.

References llvm::Empty, llvm::Instruction::getDebugLoc(), I, llvm::User::op_begin(), and llvm::User::op_end().

Referenced by llvm::InnerLoopVectorizer::createVectorizedLoopSkeleton().

getDebugLocString()

static std::string getDebugLocString ( const Loop *  L )

static

Returns

string containing a file name and a line # for the given loop.

Definition at line 779 of file LoopVectorize.cpp.

References llvm::raw_ostream::flush(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::Module::getModuleIdentifier(), llvm::BasicBlock::getParent(), llvm::GlobalValue::getParent(), and llvm::Loop::getStartLoc().

Referenced by llvm::LoopVectorizePass::processLoop().

getMemInstValueType()

static Type* getMemInstValueType ( Value *  I )

static

A helper function that returns the type of loaded or stored value.

Definition at line 280 of file LoopVectorize.cpp.

References assert(), and I.

Referenced by llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), llvm::LoopVectorizationCostModel::isLegalGatherOrScatter(), llvm::LoopVectorizationCostModel::isScalarWithPredication(), isStrideMul(), llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), llvm::InnerLoopVectorizer::vectorizeInterleaveGroup(), and llvm::InnerLoopVectorizer::vectorizeMemoryInstruction().

getReciprocalPredBlockProb()

static unsigned getReciprocalPredBlockProb ( )

static

A helper function that returns the reciprocal of the block probability of predicated blocks.

If we return X, we are assuming the predicated block will execute once for every X iterations of the loop header.

TODO: We should use actual block probability here, if available. Currently, we always assume predicated blocks have a 50% chance of executing.

Definition at line 310 of file LoopVectorize.cpp.

Referenced by llvm::LoopVectorizationCostModel::collectInstsToScalarize(), isStrideMul(), and llvm::LoopVectorizationCostModel::setCostBasedWideningDecision().

getScalarizationOverhead()

static unsigned getScalarizationOverhead ( Instruction *  I, unsigned  VF, const TargetTransformInfo &  TTI  )

static

Estimate the overhead of scalarizing an instruction.

This is a convenience wrapper for the type-based getScalarizationOverhead API.

Definition at line 3027 of file LoopVectorize.cpp.

References llvm::InnerLoopVectorizer::Cost, llvm::TargetTransformInfo::getOperandsScalarizationOverhead(), llvm::TargetTransformInfo::getScalarizationOverhead(), llvm::Value::getType(), I, llvm::Type::isVoidTy(), llvm::User::operand_values(), llvm::TargetTransformInfo::prefersVectorizedAddressing(), llvm::TargetTransformInfo::supportsEfficientVectorElementLoadStore(), ToVectorTy(), and llvm::InnerLoopVectorizer::VF.

Referenced by getVectorCallCost(), isStrideMul(), and llvm::LoopVectorizationCostModel::setCostBasedWideningDecision().

getSignedIntOrFpConstant()

static Constant* getSignedIntOrFpConstant ( Type *  Ty, int64_t  C  )

static

A helper function that returns an integer or floating-point constant with value C.

Definition at line 324 of file LoopVectorize.cpp.

References llvm::ConstantFP::get(), llvm::ConstantInt::getSigned(), and llvm::Type::isIntegerTy().

Referenced by llvm::InnerLoopVectorizer::buildScalarSteps().

getVectorCallCost()

static unsigned getVectorCallCost ( CallInst *  CI, unsigned  VF, const TargetTransformInfo &  TTI, const TargetLibraryInfo *  TLI, bool &  NeedToScalarize  )

static

Definition at line 3060 of file LoopVectorize.cpp.

References llvm::CallBase::arg_operands(), llvm::InnerLoopVectorizer::Cost, F(), llvm::CallBase::getCalledFunction(), llvm::TargetTransformInfo::getCallInstrCost(), llvm::Value::getName(), getScalarizationOverhead(), llvm::Value::getType(), llvm::TargetLibraryInfo::isFunctionVectorizable(), llvm::CallBase::isNoBuiltin(), llvm::SmallVectorTemplateBase< T >::push_back(), and ToVectorTy().

Referenced by llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), llvm::VPRecipeBuilder::tryToWiden(), and llvm::InnerLoopVectorizer::widenInstruction().

getVectorIntrinsicCost()

static unsigned getVectorIntrinsicCost ( CallInst *  CI, unsigned  VF, const TargetTransformInfo &  TTI, const TargetLibraryInfo *  TLI  )

static

Definition at line 3108 of file LoopVectorize.cpp.

References llvm::CallBase::arg_operands(), assert(), llvm::TargetTransformInfo::getIntrinsicInstrCost(), llvm::Value::getType(), llvm::getVectorIntrinsicIDForCall(), and llvm::InnerLoopVectorizer::VF.

Referenced by llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), llvm::VPRecipeBuilder::tryToWiden(), and llvm::InnerLoopVectorizer::widenInstruction().

hasIrregularType()

static bool hasIrregularType ( Type *  Ty, const DataLayout &  DL, unsigned  VF  )

static

A helper function that returns true if the given type is irregular.

The type is irregular if its allocated size doesn't equal the store size of an element of the corresponding vector type at the given vectorization factor.

Definition at line 291 of file LoopVectorize.cpp.

References llvm::VectorType::get(), llvm::DataLayout::getTypeAllocSize(), llvm::DataLayout::getTypeAllocSizeInBits(), llvm::DataLayout::getTypeSizeInBits(), and llvm::DataLayout::getTypeStoreSize().

Referenced by llvm::LoopVectorizationCostModel::memoryInstructionCanBeWidened().

isExplicitVecOuterLoop()

static bool isExplicitVecOuterLoop ( Loop *  OuterLp, OptimizationRemarkEmitter *  ORE  )

static

Definition at line 1358 of file LoopVectorize.cpp.

References assert(), llvm::dbgs(), llvm::LoopBase< BlockT, LoopT >::empty(), llvm::LoopVectorizeHints::FK_Undefined, llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::BasicBlock::getParent(), and LLVM_DEBUG.

Referenced by collectSupportedLoops().

isStrideMul()

static bool isStrideMul ( Instruction *  I, LoopVectorizationLegality *  Legal  )

static

Definition at line 5456 of file LoopVectorize.cpp.

References assert(), C, llvm::InnerLoopVectorizer::Cost, first, llvm::VectorType::get(), getAddressAccessSCEV(), llvm::TargetTransformInfo::getAddressComputationCost(), llvm::TargetTransformInfo::getGatherScatterOpCost(), llvm::TargetTransformInfo::getInterleavedMemoryOpCost(), llvm::getLoadStoreAddressSpace(), llvm::getLoadStoreAlignment(), llvm::getLoadStorePointerOperand(), llvm::TargetTransformInfo::getMaskedMemoryOpCost(), getMemInstValueType(), llvm::TargetTransformInfo::getMemoryOpCost(), llvm::TargetTransformInfo::getNumberOfParts(), llvm::Instruction::getOpcode(), llvm::User::getOperand(), getReciprocalPredBlockProb(), getScalarizationOverhead(), llvm::Type::getScalarType(), llvm::PredicatedScalarEvolution::getSE(), llvm::TargetTransformInfo::getShuffleCost(), llvm::Value::getType(), llvm::StoreInst::getValueOperand(), llvm::TargetTransformInfo::getVectorInstrCost(), llvm::LoopVectorizationLegality::hasStride(), I, llvm::LoopVectorizationLegality::isConsecutivePtr(), llvm::LoopVectorizationLegality::isMaskRequired(), llvm::LoopVectorizationLegality::isUniform(), llvm::Type::isVectorTy(), llvm::InnerLoopVectorizer::Legal, llvm::SPII::Load, llvm::InnerLoopVectorizer::PSE, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SI, llvm::TargetTransformInfo::SK_Broadcast, llvm::TargetTransformInfo::SK_Reverse, llvm::SPII::Store, ToVectorTy(), llvm::InnerLoopVectorizer::TTI, and llvm::InnerLoopVectorizer::VF.

Referenced by llvm::LoopVectorizationCostModel::setCostBasedWideningDecision().

largestIntegerVectorType()

static Type* largestIntegerVectorType ( Type *  T1, Type *  T2  )

static

Definition at line 3127 of file LoopVectorize.cpp.

References llvm::Type::getVectorElementType(), and T1.

Referenced by llvm::LoopVectorizationCostModel::setCostBasedWideningDecision().

mayDivideByZero()

static bool mayDivideByZero ( Instruction &  I )

static

A helper function for checking whether an integer division-related instruction may divide by zero (in which case it must be predicated if executed conditionally in the scalar code).

TODO: It may be worthwhile to generalize and check isKnownNonZero(). Non-zero divisors that are non compile-time constants will not be converted into multiplication, so we will still end up scalarizing the division, but can do so w/o predication.

Definition at line 3867 of file LoopVectorize.cpp.

References assert(), llvm::dyn_cast(), llvm::Instruction::getOpcode(), llvm::User::getOperand(), and llvm::ConstantInt::isZero().

Referenced by llvm::LoopVectorizationCostModel::isScalarWithPredication().

processLoopInVPlanNativePath()

static bool processLoopInVPlanNativePath ( Loop *  L, PredicatedScalarEvolution &  PSE, LoopInfo *  LI, DominatorTree *  DT, LoopVectorizationLegality *  LVL, TargetTransformInfo *  TTI, TargetLibraryInfo *  TLI, DemandedBits *  DB, AssumptionCache *  AC, OptimizationRemarkEmitter *  ORE, LoopVectorizeHints &  Hints  )

static

Definition at line 7095 of file LoopVectorize.cpp.

References assert(), llvm::dbgs(), EnableVPlanNativePath, llvm::LoopVectorizationPlanner::executePlan(), F(), llvm::LoopVectorizeHints::FK_Enabled, llvm::LoopVectorizeHints::getForce(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::LoopVectorizationLegality::getLAI(), llvm::Value::getName(), llvm::BasicBlock::getParent(), llvm::LoopVectorizeHints::getWidth(), LLVM_DEBUG, llvm::Function::optForSize(), llvm::LoopVectorizationPlanner::planInVPlanNativePath(), llvm::LoopVectorizeHints::setAlreadyVectorized(), llvm::LoopVectorizationPlanner::setBestPlan(), llvm::verifyFunction(), VPlanBuildStressTest, and llvm::VectorizationFactor::Width.

Referenced by llvm::LoopVectorizePass::processLoop().

smallestIntegerVectorType()

static Type* smallestIntegerVectorType ( Type *  T1, Type *  T2  )

static

Definition at line 3122 of file LoopVectorize.cpp.

References llvm::Type::getVectorElementType(), and T1.

Referenced by llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), and llvm::InnerLoopVectorizer::truncateToMinimalBitwidths().

STATISTIC() [1/2]

STATISTIC	(	LoopsVectorized	,
		"Number of loops vectorized"
	)

STATISTIC() [2/2]

STATISTIC	(	LoopsAnalyzed	,
		"Number of loops analyzed for vectorization"
	)

ToVectorTy()

static Type* ToVectorTy ( Type *  Scalar, unsigned  VF  )

static

A helper function for converting Scalar types to vector types.

If the incoming type is void, we return void. If the VF is 1, we return the scalar type.

Definition at line 273 of file LoopVectorize.cpp.

References llvm::VectorType::get(), and llvm::Type::isVoidTy().

Referenced by llvm::LoopVectorizationCostModel::collectInstsToScalarize(), getScalarizationOverhead(), getVectorCallCost(), llvm::LoopVectorizationCostModel::isOptimizableIVTruncate(), isStrideMul(), llvm::LoopVectorizationCostModel::setCostBasedWideningDecision(), and llvm::InnerLoopVectorizer::widenInstruction().

useMaskedInterleavedAccesses()

static bool useMaskedInterleavedAccesses ( const TargetTransformInfo &  TTI )

static

Definition at line 1958 of file LoopVectorize.cpp.

References llvm::TargetTransformInfo::enableMaskedInterleavedAccessVectorization(), and EnableMaskedInterleavedMemAccesses.

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF(), llvm::LoopVectorizationCostModel::interleavedAccessCanBeWidened(), llvm::LoopVectorizationPlanner::plan(), llvm::LoopVectorizePass::processLoop(), and llvm::InnerLoopVectorizer::vectorizeInterleaveGroup().

Variable Documentation

EnableCondStoresVectorization

cl::opt<bool> EnableCondStoresVectorization("enable-cond-stores-vec", cl::init(true), cl::Hidden, cl::desc("Enable if predication of stores during vectorization."))

static

Referenced by llvm::LoopVectorizationCostModel::selectVectorizationFactor().

EnableIndVarRegisterHeur

cl::opt<bool> EnableIndVarRegisterHeur("enable-ind-var-reg-heur", cl::init(true), cl::Hidden, cl::desc("Count the induction variable only once when interleaving"))

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

EnableInterleavedMemAccesses

cl::opt<bool> EnableInterleavedMemAccesses("enable-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on interleaved memory accesses in a loop"))

static

Referenced by llvm::LoopVectorizePass::processLoop().

EnableLoadStoreRuntimeInterleave

cl::opt<bool> EnableLoadStoreRuntimeInterleave("enable-loadstore-runtime-interleave", cl::init(true), cl::Hidden, cl::desc( "Enable runtime interleaving until load/store ports are saturated"))

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

EnableMaskedInterleavedMemAccesses

cl::opt<bool> EnableMaskedInterleavedMemAccesses("enable-masked-interleaved-mem-accesses", cl::init(false), cl::Hidden, cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"))

static

An interleave-group may need masking if it resides in a block that needs predication, or in order to mask away gaps.

Referenced by useMaskedInterleavedAccesses().

EnableVPlanNativePath

cl::opt<bool> EnableVPlanNativePath("enable-vplan-native-path", cl::init(false), cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))

Referenced by collectSupportedLoops(), llvm::InnerLoopVectorizer::emitMemRuntimeChecks(), llvm::InnerLoopVectorizer::fixVectorizedLoop(), llvm::InnerLoopVectorizer::getOrCreateVectorValue(), llvm::LoopVectorizationCostModel::getWideningDecision(), llvm::LoopVectorizationCostModel::isProfitableToScalarize(), llvm::LoopVectorizationCostModel::isScalarAfterVectorization(), llvm::LoopVectorizationCostModel::isUniformAfterVectorization(), llvm::LoopVectorizationPlanner::planInVPlanNativePath(), llvm::LoopVectorizePass::processLoop(), processLoopInVPlanNativePath(), llvm::LoopVectorizePass::run(), llvm::VPRecipeBuilder::tryToCreateRecipe(), llvm::InnerLoopVectorizer::updateAnalysis(), and llvm::InnerLoopVectorizer::widenPHIInstruction().

ForceTargetInstructionCost

cl::opt<unsigned> ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))

static

Referenced by llvm::LoopVectorizationCostModel::collectInstsToScalarize().

ForceTargetMaxScalarInterleaveFactor

cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor("force-target-max-scalar-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "scalar loops."))

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

ForceTargetMaxVectorInterleaveFactor

cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor("force-target-max-vector-interleave", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's max interleave factor for " "vectorized loops."))

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

ForceTargetNumScalarRegs

cl::opt<unsigned> ForceTargetNumScalarRegs("force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers."))

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

ForceTargetNumVectorRegs

cl::opt<unsigned> ForceTargetNumVectorRegs("force-target-num-vector-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of vector registers."))

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

LLVMLoopVectorizeFollowupAll

const char* const LLVMLoopVectorizeFollowupAll

static

Initial value:

=
    "llvm.loop.vectorize.followup_all"

Metadata attribute names

Definition at line 157 of file LoopVectorize.cpp.

Referenced by llvm::InnerLoopVectorizer::createVectorizedLoopSkeleton(), and llvm::LoopVectorizePass::processLoop().

LLVMLoopVectorizeFollowupEpilogue

const char* const LLVMLoopVectorizeFollowupEpilogue

static

Initial value:

=
    "llvm.loop.vectorize.followup_epilogue"

Definition at line 161 of file LoopVectorize.cpp.

LLVMLoopVectorizeFollowupVectorized

const char* const LLVMLoopVectorizeFollowupVectorized

static

Initial value:

=
    "llvm.loop.vectorize.followup_vectorized"

Definition at line 159 of file LoopVectorize.cpp.

LoopVectorizeWithBlockFrequency

cl::opt<bool> LoopVectorizeWithBlockFrequency("loop-vectorize-with-block-frequency", cl::init(true), cl::Hidden, cl::desc("Enable the use of the block frequency analysis to access PGO " "heuristics minimizing code growth in cold regions and being more " "aggressive in hot regions."))

static

Referenced by llvm::LoopVectorizePass::processLoop().

lv_name

const char lv_name[] = "Loop Vectorization"

static

Definition at line 6015 of file LoopVectorize.cpp.

MaximizeBandwidth

cl::opt<bool> MaximizeBandwidth("vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, cl::desc("Maximize bandwidth when selecting vectorization factor which " "will be determined by the smallest type in loop."))

static

Referenced by llvm::LoopVectorizationCostModel::computeMaxVF().

MaxNestedScalarReductionIC

cl::opt<unsigned> MaxNestedScalarReductionIC("max-nested-scalar-reduction-interleave", cl::init(2), cl::Hidden, cl::desc("The maximum interleave count to use when interleaving a scalar " "reduction in a nested loop."))

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

NumberOfStoresToPredicate

cl::opt<unsigned> NumberOfStoresToPredicate("vectorize-num-stores-pred", cl::init(1), cl::Hidden, cl::desc("Max number of stores to be predicated behind an if."))

static

The number of stores in a loop that are allowed to need predication.

Referenced by llvm::LoopVectorizationCostModel::calculateRegisterUsage().

SmallLoopCost

cl::opt<unsigned> SmallLoopCost("small-loop-cost", cl::init(20), cl::Hidden, cl::desc( "The cost of a loop that is considered 'small' by the interleaver."))

static

Referenced by llvm::LoopVectorizationCostModel::selectInterleaveCount().

TinyTripCountInterleaveThreshold

const unsigned TinyTripCountInterleaveThreshold = 128

static

We don't interleave loops with a known constant trip count below this number.

Definition at line 193 of file LoopVectorize.cpp.

TinyTripCountVectorThreshold

cl::opt<unsigned> TinyTripCountVectorThreshold("vectorizer-min-trip-count", cl::init(16), cl::Hidden, cl::desc("Loops with a constant trip count that is smaller than this " "value are vectorized only if no scalar iteration overheads " "are incurred."))

static

Loops with a known constant trip count below this number are vectorized only if no scalar iteration overheads are incurred.

Referenced by llvm::LoopVectorizePass::processLoop().

VPlanBuildStressTest

cl::opt<bool> VPlanBuildStressTest("vplan-build-stress-test", cl::init(false), cl::Hidden, cl::desc( "Build VPlan for every supported loop nest in the function and bail " "out right after the build (stress test the VPlan H-CFG construction " "in the VPlan-native vectorization path)."))

static

Referenced by collectSupportedLoops(), llvm::LoopVectorizationPlanner::planInVPlanNativePath(), and processLoopInVPlanNativePath().