LLVM
8.0.1
|
#include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/RegionInfo.h"
#include "llvm/Analysis/RegionIterator.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Transforms/Utils.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
#include <set>
#include <sstream>
Go to the source code of this file.
Namespaces | |
llvm | |
This class represents lattice values for constants. | |
Macros | |
#define | DEBUG_TYPE "chr" |
#define | CHR_DEBUG(X) LLVM_DEBUG(X) |
Functions | |
static void | parseCHRFilterFiles () |
INITIALIZE_PASS_BEGIN (ControlHeightReductionLegacyPass, "chr", "Reduce control height in the hot paths", false, false) INITIALIZE_PASS_END(ControlHeightReductionLegacyPass | |
static raw_ostream LLVM_ATTRIBUTE_UNUSED & | operator<< (raw_ostream &OS, const CHRStats &Stats) |
static raw_ostream & | operator<< (raw_ostream &OS, const CHRScope &Scope) |
static bool | shouldApply (Function &F, ProfileSummaryInfo &PSI) |
static void LLVM_ATTRIBUTE_UNUSED | dumpIR (Function &F, const char *Label, CHRStats *Stats) |
static bool | isHoistableInstructionType (Instruction *I) |
static bool | isHoistable (Instruction *I, DominatorTree &DT) |
static std::set< Value * > | getBaseValues (Value *V, DominatorTree &DT) |
static bool | checkHoistValue (Value *V, Instruction *InsertPoint, DominatorTree &DT, DenseSet< Instruction *> &Unhoistables, DenseSet< Instruction *> *HoistStops) |
static bool | checkMDProf (MDNode *MD, BranchProbability &TrueProb, BranchProbability &FalseProb) |
static BranchProbability | getCHRBiasThreshold () |
template<typename K , typename S , typename M > | |
static bool | checkBias (K *Key, BranchProbability TrueProb, BranchProbability FalseProb, S &TrueSet, S &FalseSet, M &BiasMap) |
static bool | checkBiasedBranch (BranchInst *BI, Region *R, DenseSet< Region *> &TrueBiasedRegionsGlobal, DenseSet< Region *> &FalseBiasedRegionsGlobal, DenseMap< Region *, BranchProbability > &BranchBiasMap) |
static bool | checkBiasedSelect (SelectInst *SI, Region *R, DenseSet< SelectInst *> &TrueBiasedSelectsGlobal, DenseSet< SelectInst *> &FalseBiasedSelectsGlobal, DenseMap< SelectInst *, BranchProbability > &SelectBiasMap) |
static Instruction * | getBranchInsertPoint (RegInfo &RI) |
static DenseSet< Value * > | getCHRConditionValuesForRegion (RegInfo &RI) |
static bool | shouldSplit (Instruction *InsertPoint, DenseSet< Value *> &PrevConditionValues, DenseSet< Value *> &ConditionValues, DominatorTree &DT, DenseSet< Instruction *> &Unhoistables) |
static void | getSelectsInScope (CHRScope *Scope, DenseSet< Instruction *> &Output) |
static bool | hasAtLeastTwoBiasedBranches (CHRScope *Scope) |
bool | CHRScopeSorter (CHRScope *Scope1, CHRScope *Scope2) |
static void | hoistValue (Value *V, Instruction *HoistPoint, Region *R, HoistStopMapTy &HoistStopMap, DenseSet< Instruction *> &HoistedSet, DenseSet< PHINode *> &TrivialPHIs) |
static void | hoistScopeConditions (CHRScope *Scope, Instruction *HoistPoint, DenseSet< PHINode *> &TrivialPHIs) |
static bool | negateICmpIfUsedByBranchOrSelectOnly (ICmpInst *ICmp, Instruction *ExcludedUser, CHRScope *Scope) |
static void | insertTrivialPHIs (CHRScope *Scope, BasicBlock *EntryBlock, BasicBlock *ExitBlock, DenseSet< PHINode *> &TrivialPHIs) |
static void LLVM_ATTRIBUTE_UNUSED | assertCHRRegionsHaveBiasedBranchOrSelect (CHRScope *Scope) |
static void LLVM_ATTRIBUTE_UNUSED | assertBranchOrSelectConditionHoisted (CHRScope *Scope, BasicBlock *PreEntryBlock) |
static void LLVM_ATTRIBUTE_UNUSED | dumpScopes (SmallVectorImpl< CHRScope *> &Scopes, const char *Label) |
Variables | |
static cl::opt< bool > | ForceCHR ("force-chr", cl::init(false), cl::Hidden, cl::desc("Apply CHR for all functions")) |
static cl::opt< double > | CHRBiasThreshold ("chr-bias-threshold", cl::init(0.99), cl::Hidden, cl::desc("CHR considers a branch bias greater than this ratio as biased")) |
static cl::opt< unsigned > | CHRMergeThreshold ("chr-merge-threshold", cl::init(2), cl::Hidden, cl::desc("CHR merges a group of N branches/selects where N >= this value")) |
static cl::opt< std::string > | CHRModuleList ("chr-module-list", cl::init(""), cl::Hidden, cl::desc("Specify file to retrieve the list of modules to apply CHR to")) |
static cl::opt< std::string > | CHRFunctionList ("chr-function-list", cl::init(""), cl::Hidden, cl::desc("Specify file to retrieve the list of functions to apply CHR to")) |
static StringSet | CHRModules |
static StringSet | CHRFunctions |
chr | |
Reduce control height in the hot | paths |
Reduce control height in the hot | false |
#define CHR_DEBUG | ( | X | ) | LLVM_DEBUG(X) |
Definition at line 45 of file ControlHeightReduction.cpp.
Referenced by assertBranchOrSelectConditionHoisted(), checkBiasedBranch(), checkBiasedSelect(), checkHoistValue(), dumpIR(), dumpScopes(), getBranchInsertPoint(), getSelectsInScope(), hasAtLeastTwoBiasedBranches(), hoistValue(), insertTrivialPHIs(), and shouldSplit().
#define DEBUG_TYPE "chr" |
Definition at line 43 of file ControlHeightReduction.cpp.
Referenced by assertBranchOrSelectConditionHoisted(), dumpScopes(), getBranchInsertPoint(), getSelectsInScope(), and hasAtLeastTwoBiasedBranches().
|
static |
Definition at line 1624 of file ControlHeightReduction.cpp.
References assert(), assertCHRRegionsHaveBiasedBranchOrSelect(), llvm::AMDGPUISD::BFI, CHR_DEBUG, llvm::CloneBasicBlock(), llvm::RegionBase< Tr >::contains(), llvm::BranchInst::Create(), llvm::IRBuilder< T, Inserter >::CreateAnd(), llvm::IRBuilder< T, Inserter >::CreateXor(), llvm::dbgs(), DEBUG_TYPE, llvm::User::dropAllReferences(), dumpIR(), llvm::ValueMap< KeyT, ValueT, Config >::end(), llvm::Instruction::eraseFromParent(), F(), llvm::ValueMap< KeyT, ValueT, Config >::find(), getCHRBiasThreshold(), llvm::SelectInst::getCondition(), llvm::RegionBase< Tr >::getEntry(), llvm::RegionBase< Tr >::getExit(), llvm::ConstantInt::getFalse(), llvm::BasicBlock::getInstList(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::Value::getName(), llvm::BasicBlock::getSinglePredecessor(), llvm::BranchInst::getSuccessor(), llvm::BasicBlock::getTerminator(), llvm::ConstantInt::getTrue(), llvm::Optional< T >::getValue(), hoistScopeConditions(), I, insertTrivialPHIs(), llvm::BranchInst::isUnconditional(), LLVM_ATTRIBUTE_UNUSED, llvm::LLVMContext::MD_prof, negateICmpIfUsedByBranchOrSelectOnly(), llvm::BasicBlock::phis(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::iplist_impl< IntrusiveListT, TraitsT >::push_back(), llvm::RemapInstruction(), llvm::RF_IgnoreMissingLocals, llvm::RF_NoModuleLevelChanges, llvm::SelectInst::setCondition(), llvm::BranchInst::setCondition(), llvm::Instruction::setMetadata(), SI, llvm::SmallVectorBase::size(), llvm::SplitBlock(), Stats, and std::swap().
|
static |
Definition at line 1603 of file ControlHeightReduction.cpp.
Referenced by assertBranchOrSelectConditionHoisted().
|
static |
Definition at line 625 of file ControlHeightReduction.cpp.
References getCHRBiasThreshold(), and Threshold.
Referenced by checkBiasedBranch(), and checkBiasedSelect().
|
static |
Definition at line 643 of file ControlHeightReduction.cpp.
References assert(), checkBias(), checkMDProf(), CHR_DEBUG, llvm::dbgs(), llvm::RegionBase< Tr >::getExit(), llvm::Instruction::getMetadata(), llvm::BranchInst::getSuccessor(), llvm::BranchInst::isConditional(), llvm::LLVMContext::MD_prof, and std::swap().
Referenced by getBranchInsertPoint().
|
static |
Definition at line 674 of file ControlHeightReduction.cpp.
References checkBias(), checkMDProf(), CHR_DEBUG, llvm::dbgs(), llvm::Instruction::getMetadata(), and llvm::LLVMContext::MD_prof.
Referenced by getBranchInsertPoint().
|
static |
Definition at line 548 of file ControlHeightReduction.cpp.
References assert(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::begin(), CHR_DEBUG, llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::count(), llvm::dbgs(), llvm::DominatorTree::dominates(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::end(), llvm::DominatorTreeBase< NodeT, IsPostDom >::getNode(), llvm::Instruction::getParent(), I, llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::insert(), and isHoistable().
Referenced by getBranchInsertPoint(), hasAtLeastTwoBiasedBranches(), and shouldSplit().
|
static |
Definition at line 592 of file ControlHeightReduction.cpp.
References assert(), llvm::BranchProbability::getBranchProbability(), llvm::MDNode::getNumOperands(), llvm::MDNode::getOperand(), llvm::MDString::getString(), llvm::ConstantInt::getValue(), and llvm::APInt::getZExtValue().
Referenced by checkBiasedBranch(), and checkBiasedSelect().
bool CHRScopeSorter | ( | CHRScope * | Scope1, |
CHRScope * | Scope2 | ||
) |
Definition at line 1412 of file ControlHeightReduction.cpp.
References llvm::SmallVectorTemplateCommon< T >::begin(), llvm::copy(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorBase::size().
|
static |
Definition at line 456 of file ControlHeightReduction.cpp.
References assert(), CHR_DEBUG, llvm::dbgs(), llvm::Value::dump(), llvm::Module::getName(), llvm::Value::getName(), llvm::GlobalValue::getParent(), and print().
Referenced by assertBranchOrSelectConditionHoisted(), and dumpScopes().
|
static |
Definition at line 1954 of file ControlHeightReduction.cpp.
References llvm::AMDGPUISD::BFI, CHR_DEBUG, llvm::dbgs(), DEBUG_TYPE, dumpIR(), llvm::SmallVectorBase::empty(), F(), runOnFunction(), shouldApply(), llvm::SmallVectorBase::size(), and Stats.
|
static |
Definition at line 516 of file ControlHeightReduction.cpp.
References I, and isHoistable().
Referenced by shouldSplit().
|
static |
Definition at line 694 of file ControlHeightReduction.cpp.
References assert(), llvm::RegionBase< Tr >::begin(), llvm::MCID::Branch, checkBiasedBranch(), checkBiasedSelect(), checkHoistValue(), CHR_DEBUG, llvm::RegionBase< Tr >::contains(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::count(), llvm::dbgs(), DEBUG_TYPE, llvm::dyn_cast(), E, llvm::RegionBase< Tr >::elements(), llvm::RegionBase< Tr >::end(), llvm::SelectInst::getCondition(), llvm::RegionBase< Tr >::getEntry(), llvm::RegionBase< Tr >::getExit(), llvm::Value::getName(), llvm::RegionBase< Tr >::getNameStr(), llvm::Instruction::getParent(), llvm::BasicBlock::getTerminator(), I, llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::insert(), llvm::RegionBase< Tr >::isTopLevelRegion(), llvm::predecessors(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::remove_if(), SI, llvm::SmallVectorBase::size(), and Stats.
Referenced by getSelectsInScope().
|
static |
Definition at line 615 of file ControlHeightReduction.cpp.
References CHRBiasThreshold, and llvm::BranchProbability::getBranchProbability().
Referenced by assertBranchOrSelectConditionHoisted(), and checkBias().
Definition at line 1020 of file ControlHeightReduction.cpp.
References llvm::SelectInst::getCondition(), llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), and SI.
Referenced by getSelectsInScope().
|
static |
Definition at line 1100 of file ControlHeightReduction.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T >::begin(), CHR_DEBUG, llvm::dbgs(), DEBUG_TYPE, llvm::SmallVectorBase::empty(), llvm::SmallVectorTemplateCommon< T >::end(), getBranchInsertPoint(), getCHRConditionValuesForRegion(), llvm::RegionBase< Tr >::getNameStr(), I, llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::insert(), llvm::SmallVectorImpl< T >::insert(), llvm_unreachable, llvm::SmallVectorTemplateBase< T >::push_back(), shouldSplit(), SI, llvm::SmallVectorBase::size(), and Split().
|
static |
Definition at line 1309 of file ControlHeightReduction.cpp.
References assert(), checkHoistValue(), CHR_DEBUG, CHRMergeThreshold, llvm::dbgs(), DEBUG_TYPE, llvm::SelectInst::getCondition(), llvm::RegionBase< Tr >::getEntry(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::insert(), llvm::SmallVectorTemplateBase< T >::push_back(), and SI.
|
static |
Definition at line 1459 of file ControlHeightReduction.cpp.
References llvm::SelectInst::getCondition(), llvm::RegionBase< Tr >::getEntry(), hoistValue(), and SI.
Referenced by assertBranchOrSelectConditionHoisted().
|
static |
Definition at line 1425 of file ControlHeightReduction.cpp.
References assert(), CHR_DEBUG, llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::count(), llvm::dbgs(), I, llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::insert(), isHoistableInstructionType(), and IT.
Referenced by hoistScopeConditions().
INITIALIZE_PASS_BEGIN | ( | ControlHeightReductionLegacyPass | , |
"chr" | , | ||
"Reduce control height in the hot paths" | , | ||
false | , | ||
false | |||
) |
Referenced by parseCHRFilterFiles().
|
static |
Definition at line 1536 of file ControlHeightReduction.cpp.
References llvm::PHINode::addIncoming(), CHR_DEBUG, llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::count(), llvm::PHINode::Create(), llvm::dbgs(), llvm::BasicBlock::front(), I, llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::insert(), LLVM_ATTRIBUTE_UNUSED, llvm::pred_begin(), llvm::pred_end(), llvm::predecessors(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorBase::size(), and Users.
Referenced by assertBranchOrSelectConditionHoisted().
|
static |
Definition at line 505 of file ControlHeightReduction.cpp.
References isHoistableInstructionType(), and llvm::isSafeToSpeculativelyExecute().
Referenced by checkHoistValue(), and getBaseValues().
|
static |
Definition at line 496 of file ControlHeightReduction.cpp.
References I.
Referenced by hoistValue(), and isHoistable().
|
static |
Definition at line 1484 of file ControlHeightReduction.cpp.
References assert(), llvm::CmpInst::getInversePredicate(), llvm::CmpInst::getPredicate(), llvm_unreachable, llvm::CmpInst::setPredicate(), SI, and llvm::Value::users().
Referenced by assertBranchOrSelectConditionHoisted().
|
inlinestatic |
Definition at line 430 of file ControlHeightReduction.cpp.
|
inlinestatic |
Definition at line 437 of file ControlHeightReduction.cpp.
|
static |
Definition at line 69 of file ControlHeightReduction.cpp.
References llvm::AnalysisUsage::addPreserved(), llvm::AnalysisUsage::addRequired(), CHRFunctionList, CHRModuleList, llvm::errs(), F(), llvm::MemoryBuffer::getFile(), llvm::PassRegistry::getPassRegistry(), INITIALIZE_PASS_BEGIN(), INITIALIZE_PASS_DEPENDENCY, INITIALIZE_PASS_END(), llvm::initializeControlHeightReductionLegacyPassPass(), llvm::StringSet< AllocatorTy >::insert(), runOnFunction(), and llvm::StringRef::split().
Referenced by llvm::ControlHeightReductionPass::ControlHeightReductionPass().
|
static |
Definition at line 442 of file ControlHeightReduction.cpp.
References assert(), CHRFunctionList, CHRFunctions, CHRModuleList, CHRModules, llvm::StringMap< ValueTy, AllocatorTy >::count(), ForceCHR, llvm::Module::getName(), llvm::Value::getName(), llvm::GlobalValue::getParent(), llvm::ProfileSummaryInfo::hasProfileSummary(), and llvm::ProfileSummaryInfo::isFunctionEntryHot().
Referenced by dumpScopes().
|
static |
Definition at line 1039 of file ControlHeightReduction.cpp.
References assert(), checkHoistValue(), CHR_DEBUG, llvm::dbgs(), and getBaseValues().
Referenced by getSelectsInScope().
chr |
Definition at line 135 of file ControlHeightReduction.cpp.
|
static |
Referenced by getCHRBiasThreshold().
|
static |
Referenced by parseCHRFilterFiles(), and shouldApply().
|
static |
Definition at line 67 of file ControlHeightReduction.cpp.
Referenced by shouldApply().
|
static |
Referenced by hasAtLeastTwoBiasedBranches().
|
static |
Referenced by parseCHRFilterFiles(), and shouldApply().
|
static |
Definition at line 66 of file ControlHeightReduction.cpp.
Referenced by shouldApply().
Reduce control height in the hot false |
Definition at line 135 of file ControlHeightReduction.cpp.
|
static |
Referenced by shouldApply().
Reduce control height in the hot paths |
Definition at line 135 of file ControlHeightReduction.cpp.