LLVM  8.0.1
Classes | Namespaces | Macros | Typedefs | Enumerations | Functions | Variables
PGOInstrumentation.cpp File Reference
#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
#include "CFGMST.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/ADT/iterator.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/IndirectCallVisitor.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/Comdat.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfileSummary.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
#include "llvm/Support/BranchProbability.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/JamCRC.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <memory>
#include <numeric>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>

Go to the source code of this file.

Classes

struct  llvm::GraphTraits< PGOUseFunc * >
 
struct  llvm::DOTGraphTraits< PGOUseFunc * >
 

Namespaces

 llvm
 This class represents lattice values for constants.
 

Macros

#define DEBUG_TYPE   "pgo-instrumentation"
 

Typedefs

using ProfileCount = Function::ProfileCount
 

Enumerations

enum  VisitMode
 The select instruction visitor plays three roles specified by the mode. More...
 

Functions

 STATISTIC (NumOfPGOInstrument, "Number of edges instrumented.")
 
 STATISTIC (NumOfPGOSelectInsts, "Number of select instruction instrumented.")
 
 STATISTIC (NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.")
 
 STATISTIC (NumOfPGOEdge, "Number of edges.")
 
 STATISTIC (NumOfPGOBB, "Number of basic-blocks.")
 
 STATISTIC (NumOfPGOSplit, "Number of critical edge splits.")
 
 STATISTIC (NumOfPGOFunc, "Number of functions having valid profile counts.")
 
 STATISTIC (NumOfPGOMismatch, "Number of functions having mismatch profile.")
 
 STATISTIC (NumOfPGOMissing, "Number of functions without profile.")
 
 STATISTIC (NumOfPGOICall, "Number of indirect call value instrumentations.")
 
static std::string getBranchCondString (Instruction *TI)
 
 INITIALIZE_PASS_BEGIN (PGOInstrumentationGenLegacyPass, "pgo-instr-gen", "PGO instrumentation.", false, false) INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass
 
 INITIALIZE_PASS_BEGIN (PGOInstrumentationUseLegacyPass, "pgo-instr-use", "Read PGO instrumentation profile.", false, false) INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass
 
static bool canRenameComdat (Function &F, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers)
 
static void instrumentOneFunc (Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers)
 
static uint64_t sumEdgeCount (const ArrayRef< PGOUseEdge *> Edges)
 
static bool isIndirectBrTarget (BasicBlock *BB)
 
static void createIRLevelProfileFlagVariable (Module &M)
 
static void collectComdatMembers (Module &M, std::unordered_multimap< Comdat *, GlobalValue *> &ComdatMembers)
 
static bool InstrumentAllFunctions (Module &M, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI)
 
static bool annotateAllFunctions (Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI)
 
static std::string getSimpleNodeName (const BasicBlock *Node)
 
void llvm::setIrrLoopHeaderMetadata (Module *M, Instruction *TI, uint64_t Count)
 

Variables

static cl::opt< std::string > PGOTestProfileFile ("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
 
static cl::opt< std::string > PGOTestProfileRemappingFile ("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
 
static cl::opt< boolDisableValueProfiling ("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
 
static cl::opt< unsignedMaxNumAnnotations ("icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of annotations for a single indirect " "call callsite"))
 
static cl::opt< unsignedMaxNumMemOPAnnotations ("memop-max-annotations", cl::init(4), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
 
static cl::opt< boolDoComdatRenaming ("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
 
static cl::opt< boolPGOWarnMissing ("pgo-warn-missing-function", cl::init(false), cl::Hidden, cl::desc("Use this option to turn on/off " "warnings about missing profile data for " "functions."))
 
static cl::opt< boolNoPGOWarnMismatch ("no-pgo-warn-mismatch", cl::init(false), cl::Hidden, cl::desc("Use this option to turn off/on " "warnings about profile cfg mismatch."))
 
static cl::opt< boolNoPGOWarnMismatchComdat ("no-pgo-warn-mismatch-comdat", cl::init(true), cl::Hidden, cl::desc("The option is used to turn on/off " "warnings about hash mismatch for comdat " "functions."))
 
static cl::opt< boolPGOInstrSelect ("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
 
static cl::opt< PGOViewCountsTypePGOViewRawCounts ("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
 
static cl::opt< boolPGOInstrMemOP ("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
 
static cl::opt< boolEmitBranchProbability ("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
 
cl::opt< PGOViewCountsType > PGOViewCounts
 
cl::opt< std::string > ViewBlockFreqFuncName
 
pgo instr gen
 
pgo instr PGO instrumentation
 
pgo instr PGO false
 
pgo instr use
 
pgo instr Read PGO instrumentation profile
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "pgo-instrumentation"

Definition at line 124 of file PGOInstrumentation.cpp.

Referenced by llvm::setProfMetadata().

Typedef Documentation

◆ ProfileCount

Definition at line 122 of file PGOInstrumentation.cpp.

Enumeration Type Documentation

◆ VisitMode

enum VisitMode

The select instruction visitor plays three roles specified by the mode.

In VM_counting mode, it simply counts the number of select instructions. In VM_instrument mode, it inserts code to count the number times TrueValue of select is taken. In VM_annotate mode, it reads the profile data and annotate the select instruction with metadata.

Definition at line 283 of file PGOInstrumentation.cpp.

Function Documentation

◆ annotateAllFunctions()

static bool annotateAllFunctions ( Module M,
StringRef  ProfileFileName,
StringRef  ProfileRemappingFileName,
function_ref< BranchProbabilityInfo *(Function &)>  LookupBPI,
function_ref< BlockFrequencyInfo *(Function &)>  LookupBFI 
)
static

◆ canRenameComdat()

static bool canRenameComdat ( Function F,
std::unordered_multimap< Comdat *, GlobalValue *> &  ComdatMembers 
)
static

◆ collectComdatMembers()

static void collectComdatMembers ( Module M,
std::unordered_multimap< Comdat *, GlobalValue *> &  ComdatMembers 
)
static

Definition at line 1375 of file PGOInstrumentation.cpp.

References C, DoComdatRenaming, and F().

Referenced by annotateAllFunctions(), and InstrumentAllFunctions().

◆ createIRLevelProfileFlagVariable()

static void createIRLevelProfileFlagVariable ( Module M)
static

◆ getBranchCondString()

static std::string getBranchCondString ( Instruction TI)
static

◆ getSimpleNodeName()

static std::string getSimpleNodeName ( const BasicBlock Node)
static

Definition at line 1590 of file PGOInstrumentation.cpp.

◆ INITIALIZE_PASS_BEGIN() [1/2]

INITIALIZE_PASS_BEGIN ( PGOInstrumentationGenLegacyPass  ,
"pgo-instr-gen ,
"PGO instrumentation."  ,
false  ,
false   
)

◆ INITIALIZE_PASS_BEGIN() [2/2]

INITIALIZE_PASS_BEGIN ( PGOInstrumentationUseLegacyPass  ,
"pgo-instr-use ,
"Read PGO instrumentation profile."  ,
false  ,
false   
)

◆ InstrumentAllFunctions()

static bool InstrumentAllFunctions ( Module M,
function_ref< BranchProbabilityInfo *(Function &)>  LookupBPI,
function_ref< BlockFrequencyInfo *(Function &)>  LookupBFI 
)
static

◆ instrumentOneFunc()

static void instrumentOneFunc ( Function F,
Module M,
BranchProbabilityInfo BPI,
BlockFrequencyInfo BFI,
std::unordered_multimap< Comdat *, GlobalValue *> &  ComdatMembers 
)
static

◆ isIndirectBrTarget()

static bool isIndirectBrTarget ( BasicBlock BB)
static

◆ STATISTIC() [1/10]

STATISTIC ( NumOfPGOInstrument  ,
"Number of edges instrumented."   
)

◆ STATISTIC() [2/10]

STATISTIC ( NumOfPGOSelectInsts  ,
"Number of select instruction instrumented."   
)

◆ STATISTIC() [3/10]

STATISTIC ( NumOfPGOMemIntrinsics  ,
"Number of mem intrinsics instrumented."   
)

◆ STATISTIC() [4/10]

STATISTIC ( NumOfPGOEdge  ,
"Number of edges."   
)

◆ STATISTIC() [5/10]

STATISTIC ( NumOfPGOBB  ,
"Number of basic-blocks."   
)

◆ STATISTIC() [6/10]

STATISTIC ( NumOfPGOSplit  ,
"Number of critical edge splits."   
)

◆ STATISTIC() [7/10]

STATISTIC ( NumOfPGOFunc  ,
"Number of functions having valid profile counts."   
)

◆ STATISTIC() [8/10]

STATISTIC ( NumOfPGOMismatch  ,
"Number of functions having mismatch profile."   
)

◆ STATISTIC() [9/10]

STATISTIC ( NumOfPGOMissing  ,
"Number of functions without profile."   
)

◆ STATISTIC() [10/10]

STATISTIC ( NumOfPGOICall  ,
"Number of indirect call value instrumentations."   
)

◆ sumEdgeCount()

static uint64_t sumEdgeCount ( const ArrayRef< PGOUseEdge *>  Edges)
static

Variable Documentation

◆ DisableValueProfiling

cl::opt<bool> DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))
static

◆ DoComdatRenaming

cl::opt<bool> DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))
static

◆ EmitBranchProbability

cl::opt<bool> EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))
static

Referenced by llvm::setProfMetadata().

◆ false

pgo instr Read PGO instrumentation false

Definition at line 435 of file PGOInstrumentation.cpp.

◆ gen

pgo instr gen

Definition at line 435 of file PGOInstrumentation.cpp.

◆ instrumentation

pgo instr PGO instrumentation

Definition at line 435 of file PGOInstrumentation.cpp.

◆ MaxNumAnnotations

cl::opt<unsigned> MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of annotations for a single indirect " "call callsite"))
static

Referenced by isIndirectBrTarget().

◆ MaxNumMemOPAnnotations

cl::opt<unsigned> MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::ZeroOrMore, cl::desc("Max number of preicise value annotations for a single memop" "intrinsic"))
static

Referenced by isIndirectBrTarget().

◆ NoPGOWarnMismatch

cl::opt<bool> NoPGOWarnMismatch("no-pgo-warn-mismatch", cl::init(false), cl::Hidden, cl::desc("Use this option to turn off/on " "warnings about profile cfg mismatch."))
static

◆ NoPGOWarnMismatchComdat

cl::opt<bool> NoPGOWarnMismatchComdat("no-pgo-warn-mismatch-comdat", cl::init(true), cl::Hidden, cl::desc("The option is used to turn on/off " "warnings about hash mismatch for comdat " "functions."))
static

◆ PGOInstrMemOP

cl::opt<bool> PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))
static

Referenced by isIndirectBrTarget().

◆ PGOInstrSelect

cl::opt<bool> PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))
static

Referenced by isIndirectBrTarget().

◆ PGOTestProfileFile

cl::opt<std::string> PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is" "mainly for test purpose."))
static

◆ PGOTestProfileRemappingFile

cl::opt<std::string> PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))
static

◆ PGOViewCounts

cl::opt<PGOViewCountsType> PGOViewCounts

◆ PGOViewRawCounts

cl::opt<PGOViewCountsType> PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))
static

Referenced by annotateAllFunctions().

◆ PGOWarnMissing

cl::opt<bool> PGOWarnMissing("pgo-warn-missing-function", cl::init(false), cl::Hidden, cl::desc("Use this option to turn on/off " "warnings about missing profile data for " "functions."))
static

◆ profile

pgo instr Read PGO instrumentation profile

Definition at line 448 of file PGOInstrumentation.cpp.

◆ use

pgo instr use

◆ ViewBlockFreqFuncName

cl::opt<std::string> ViewBlockFreqFuncName