LLVM
8.0.1
|
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "CFGMST.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/IndirectCallVisitor.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Comdat.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/JamCRC.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <memory>
#include <numeric>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
Go to the source code of this file.
Classes | |
struct | llvm::GraphTraits< PGOUseFunc * > |
struct | llvm::DOTGraphTraits< PGOUseFunc * > |
Namespaces | |
llvm | |
This class represents lattice values for constants. | |
Macros | |
#define | DEBUG_TYPE "pgo-instrumentation" |
Typedefs | |
using | ProfileCount = Function::ProfileCount |
Enumerations | |
enum | VisitMode |
The select instruction visitor plays three roles specified by the mode. More... | |
Functions | |
STATISTIC (NumOfPGOInstrument, "Number of edges instrumented.") | |
STATISTIC (NumOfPGOSelectInsts, "Number of select instruction instrumented.") | |
STATISTIC (NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.") | |
STATISTIC (NumOfPGOEdge, "Number of edges.") | |
STATISTIC (NumOfPGOBB, "Number of basic-blocks.") | |
STATISTIC (NumOfPGOSplit, "Number of critical edge splits.") | |
STATISTIC (NumOfPGOFunc, "Number of functions having valid profile counts.") | |
STATISTIC (NumOfPGOMismatch, "Number of functions having mismatch profile.") | |
STATISTIC (NumOfPGOMissing, "Number of functions without profile.") | |
STATISTIC (NumOfPGOICall, "Number of indirect call value instrumentations.") | |
static std::string | getBranchCondString (Instruction *TI) |
INITIALIZE_PASS_BEGIN (PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass | |
INITIALIZE_PASS_BEGIN (PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass | |
static bool | canRenameComdat (Function &F, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers) |
static void | instrumentOneFunc (Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers) |
static uint64_t | sumEdgeCount (const ArrayRef< PGOUseEdge *> Edges) |
static bool | isIndirectBrTarget (BasicBlock *BB) |
static void | createIRLevelProfileFlagVariable (Module &M) |
static void | collectComdatMembers (Module &M, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers) |
static bool | InstrumentAllFunctions (Module &M, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI) |
static bool | annotateAllFunctions (Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI) |
static std::string | getSimpleNodeName (const BasicBlock *Node) |
void | llvm::setIrrLoopHeaderMetadata (Module *M, Instruction *TI, uint64_t Count) |
Variables | |
static cl::opt< std::string > | PGOTestProfileFile ("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose.")) |
static cl::opt< std::string > | PGOTestProfileRemappingFile ("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose.")) |
static cl::opt< bool > | DisableValueProfiling ("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling")) |
static cl::opt< unsigned > | MaxNumAnnotations ("icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of annotations for a single indirect " "call callsite")) |
static cl::opt< unsigned > | MaxNumMemOPAnnotations ("memop-max-annotations", cl::init(4), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic")) |
static cl::opt< bool > | DoComdatRenaming ("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner")) |
static cl::opt< bool > | PGOWarnMissing ("pgo-warn-missing-function", cl::init(false), cl::Hidden, cl::desc("Use this option to turn on/off " "warnings about missing profile data for " "functions.")) |
static cl::opt< bool > | NoPGOWarnMismatch ("no-pgo-warn-mismatch", cl::init(false), cl::Hidden, cl::desc("Use this option to turn off/on " "warnings about profile cfg mismatch.")) |
static cl::opt< bool > | NoPGOWarnMismatchComdat ("no-pgo-warn-mismatch-comdat", cl::init(true), cl::Hidden, cl::desc("The option is used to turn on/off " "warnings about hash mismatch for comdat " "functions.")) |
static cl::opt< bool > | PGOInstrSelect ("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. ")) |
static cl::opt< PGOViewCountsType > | PGOViewRawCounts ("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text."))) |
static cl::opt< bool > | PGOInstrMemOP ("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling.")) |
static cl::opt< bool > | EmitBranchProbability ("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation")) |
cl::opt< PGOViewCountsType > | PGOViewCounts |
cl::opt< std::string > | ViewBlockFreqFuncName |
pgo instr | gen |
pgo instr PGO | instrumentation |
pgo instr PGO | false |
pgo instr | use |
pgo instr Read PGO instrumentation | profile |
#define DEBUG_TYPE "pgo-instrumentation" |
Definition at line 124 of file PGOInstrumentation.cpp.
Referenced by llvm::setProfMetadata().
using ProfileCount = Function::ProfileCount |
Definition at line 122 of file PGOInstrumentation.cpp.
enum VisitMode |
The select instruction visitor plays three roles specified by the mode.
In VM_counting
mode, it simply counts the number of select instructions. In VM_instrument
mode, it inserts code to count the number times TrueValue of select is taken. In VM_annotate
mode, it reads the profile data and annotate the select instruction with metadata.
Definition at line 283 of file PGOInstrumentation.cpp.
|
static |
Definition at line 1438 of file PGOInstrumentation.cpp.
References llvm::AMDGPUISD::BFI, llvm::Attribute::Cold, collectComdatMembers(), llvm::IndexedInstrProfReader::create(), llvm::StringRef::data(), llvm::dbgs(), F(), llvm::Module::getContext(), llvm::handleAllErrors(), llvm::Attribute::InlineHint, LLVM_DEBUG, llvm::ErrorInfoBase::message(), llvm::Function::PCT_Real, llvm::PGOVCT_Graph, llvm::PGOVCT_None, llvm::PGOVCT_Text, PGOViewCounts, PGOViewRawCounts, llvm::SplitIndirectBrCriticalEdges(), ViewBlockFreqFuncName, ViewGraph(), and llvm::WriteGraph().
Referenced by llvm::PGOInstrumentationUse::run().
|
static |
Definition at line 613 of file PGOInstrumentation.cpp.
References assert(), llvm::GlobalValue::AvailableExternallyLinkage, C, llvm::canRenameComdatFunc(), llvm::GlobalAlias::create(), llvm::dbgs(), DoComdatRenaming, llvm::dyn_cast(), E, F(), llvm::GlobalObject::getComdat(), llvm::Comdat::getName(), llvm::Instruction::getNumSuccessors(), llvm::Module::getOrInsertComdat(), llvm::Comdat::getSelectionKind(), llvm::GetSuccessorNumber(), llvm::BasicBlock::getTerminator(), llvm::GlobalValue::LinkOnceODRLinkage, LLVM_DEBUG, llvm::make_range(), llvm::GlobalObject::setComdat(), llvm::Comdat::setSelectionKind(), llvm::SplitCriticalEdge(), llvm::Twine::str(), and llvm::GlobalValue::WeakAnyLinkage.
|
static |
Definition at line 1375 of file PGOInstrumentation.cpp.
References C, DoComdatRenaming, and F().
Referenced by annotateAllFunctions(), and InstrumentAllFunctions().
|
static |
Definition at line 1357 of file PGOInstrumentation.cpp.
References llvm::GlobalValue::DefaultVisibility, llvm::GlobalValue::ExternalLinkage, llvm::Module::getContext(), llvm::Type::getInt64Ty(), llvm::Constant::getIntegerValue(), llvm::Module::getOrInsertComdat(), llvm::Module::getTargetTriple(), and llvm::GlobalValue::WeakAnyLinkage.
Referenced by InstrumentAllFunctions().
|
static |
Definition at line 245 of file PGOInstrumentation.cpp.
References llvm::dyn_cast(), llvm::raw_ostream::flush(), llvm::BranchInst::getCondition(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::CmpInst::getPredicateName(), llvm::Value::getType(), llvm::BranchInst::isConditional(), llvm::ConstantInt::isMinusOne(), llvm::ConstantInt::isOne(), llvm::ConstantInt::isZero(), and llvm::Type::print().
Referenced by llvm::setProfMetadata().
|
static |
Definition at line 1590 of file PGOInstrumentation.cpp.
INITIALIZE_PASS_BEGIN | ( | PGOInstrumentationGenLegacyPass | , |
"pgo-instr-gen" | , | ||
"PGO instrumentation." | , | ||
false | , | ||
false | |||
) |
Referenced by llvm::createPGOInstrumentationGenLegacyPass().
INITIALIZE_PASS_BEGIN | ( | PGOInstrumentationUseLegacyPass | , |
"pgo-instr-use" | , | ||
"Read PGO instrumentation profile." | , | ||
false | , | ||
false | |||
) |
|
static |
Definition at line 1391 of file PGOInstrumentation.cpp.
References llvm::AMDGPUISD::BFI, collectComdatMembers(), createIRLevelProfileFlagVariable(), F(), and instrumentOneFunc().
Referenced by llvm::PGOInstrumentationGen::run().
|
static |
Definition at line 722 of file PGOInstrumentation.cpp.
References assert(), C, Callee, llvm::IRBuilder< T, Inserter >::CreateCall(), llvm::IRBuilder< T, Inserter >::CreatePtrToInt(), llvm::dbgs(), DisableValueProfiling, llvm::BasicBlock::end(), llvm::ConstantExpr::getBitCast(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getCalledValue(), llvm::Module::getContext(), llvm::Intrinsic::getDeclaration(), llvm::BasicBlock::getFirstInsertionPt(), llvm::IRBuilderBase::GetInsertPoint(), llvm::IRBuilderBase::getInt32(), llvm::IRBuilderBase::getInt64(), llvm::IRBuilderBase::getInt64Ty(), llvm::Type::getInt8PtrTy(), I, llvm::Intrinsic::instrprof_increment, llvm::Intrinsic::instrprof_value_profile, LLVM_DEBUG, llvm::SplitIndirectBrCriticalEdges(), and llvm::RISCVFenceField::W.
Referenced by InstrumentAllFunctions().
|
static |
Definition at line 1202 of file PGOInstrumentation.cpp.
References llvm::annotateValueSite(), assert(), llvm::AMDGPUISD::BFI, llvm::IRBuilder< T, Inserter >::CreateCall(), llvm::createPGOFuncNameMetadata(), llvm::IRBuilder< T, Inserter >::CreateZExt(), llvm::IRBuilder< T, Inserter >::CreateZExtOrTrunc(), llvm::dbgs(), DisableValueProfiling, llvm::DS_Warning, E, F(), llvm::ConstantExpr::getBitCast(), llvm::SelectInst::getCondition(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::getInt32(), llvm::IRBuilderBase::getInt64(), llvm::IRBuilderBase::getInt64Ty(), llvm::IRBuilderBase::getInt8PtrTy(), llvm::MemIntrinsicBase< Derived >::getLength(), llvm::Instruction::getParent(), llvm::Value::getType(), I, llvm::Intrinsic::instrprof_increment_step, llvm::Intrinsic::instrprof_value_profile, llvm::Type::isVectorTy(), Kind, LLVM_DEBUG, llvm_unreachable, llvm::max(), MaxNumAnnotations, MaxNumMemOPAnnotations, MI, Mode, PGOInstrMemOP, PGOInstrSelect, llvm::pred_begin(), llvm::pred_end(), llvm::setIrrLoopHeaderMetadata(), llvm::setProfMetadata(), and SI.
STATISTIC | ( | NumOfPGOInstrument | , |
"Number of edges instrumented." | |||
) |
STATISTIC | ( | NumOfPGOSelectInsts | , |
"Number of select instruction instrumented." | |||
) |
STATISTIC | ( | NumOfPGOMemIntrinsics | , |
"Number of mem intrinsics instrumented." | |||
) |
STATISTIC | ( | NumOfPGOEdge | , |
"Number of edges." | |||
) |
STATISTIC | ( | NumOfPGOBB | , |
"Number of basic-blocks." | |||
) |
STATISTIC | ( | NumOfPGOSplit | , |
"Number of critical edge splits." | |||
) |
STATISTIC | ( | NumOfPGOMismatch | , |
"Number of functions having mismatch profile." | |||
) |
STATISTIC | ( | NumOfPGOMissing | , |
"Number of functions without profile." | |||
) |
STATISTIC | ( | NumOfPGOICall | , |
"Number of indirect call value instrumentations." | |||
) |
Definition at line 839 of file PGOInstrumentation.cpp.
References assert(), llvm::AMDGPUISD::BFI, F(), llvm::IndexedInstrProfReader::getInstrProfRecord(), llvm::handleAllErrors(), I, Kind, llvm_unreachable, llvm::BranchProbability::scale(), setBranchWeights(), and llvm::Expected< T >::takeError().
|
static |
Referenced by instrumentOneFunc(), and isIndirectBrTarget().
|
static |
Referenced by canRenameComdat(), and collectComdatMembers().
|
static |
Referenced by llvm::setProfMetadata().
pgo instr Read PGO instrumentation false |
Definition at line 435 of file PGOInstrumentation.cpp.
pgo instr gen |
Definition at line 435 of file PGOInstrumentation.cpp.
pgo instr PGO instrumentation |
Definition at line 435 of file PGOInstrumentation.cpp.
|
static |
Referenced by isIndirectBrTarget().
|
static |
Referenced by isIndirectBrTarget().
|
static |
|
static |
|
static |
Referenced by isIndirectBrTarget().
|
static |
Referenced by isIndirectBrTarget().
|
static |
Referenced by llvm::PGOInstrumentationUse::PGOInstrumentationUse().
|
static |
Referenced by llvm::PGOInstrumentationUse::PGOInstrumentationUse().
cl::opt<PGOViewCountsType> PGOViewCounts |
Referenced by annotateAllFunctions(), and llvm::getGVDT().
|
static |
Referenced by annotateAllFunctions().
|
static |
pgo instr Read PGO instrumentation profile |
Definition at line 448 of file PGOInstrumentation.cpp.
pgo instr use |
Definition at line 448 of file PGOInstrumentation.cpp.
Referenced by llvm::GEPOperator::countNonConstantIndices(), llvm::CodeExtractor::findInputsOutputs(), and llvm::HexagonSubtarget::getHexagonArchVersion().
cl::opt<std::string> ViewBlockFreqFuncName |
Referenced by annotateAllFunctions(), and llvm::BlockFrequencyInfo::calculate().