|
LLVM
8.0.1
|
#include "llvm/ADT/ArrayRef.h"#include "llvm/ADT/BitVector.h"#include "llvm/ADT/DenseMap.h"#include "llvm/ADT/MapVector.h"#include "llvm/ADT/PriorityQueue.h"#include "llvm/ADT/SetVector.h"#include "llvm/ADT/SmallPtrSet.h"#include "llvm/ADT/SmallSet.h"#include "llvm/ADT/SmallVector.h"#include "llvm/ADT/Statistic.h"#include "llvm/ADT/iterator_range.h"#include "llvm/Analysis/AliasAnalysis.h"#include "llvm/Analysis/MemoryLocation.h"#include "llvm/Analysis/ValueTracking.h"#include "llvm/CodeGen/DFAPacketizer.h"#include "llvm/CodeGen/LiveIntervals.h"#include "llvm/CodeGen/MachineBasicBlock.h"#include "llvm/CodeGen/MachineDominators.h"#include "llvm/CodeGen/MachineFunction.h"#include "llvm/CodeGen/MachineFunctionPass.h"#include "llvm/CodeGen/MachineInstr.h"#include "llvm/CodeGen/MachineInstrBuilder.h"#include "llvm/CodeGen/MachineLoopInfo.h"#include "llvm/CodeGen/MachineMemOperand.h"#include "llvm/CodeGen/MachineOperand.h"#include "llvm/CodeGen/MachinePipeliner.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/RegisterPressure.h"#include "llvm/CodeGen/ScheduleDAG.h"#include "llvm/CodeGen/ScheduleDAGMutation.h"#include "llvm/CodeGen/TargetOpcodes.h"#include "llvm/CodeGen/TargetRegisterInfo.h"#include "llvm/CodeGen/TargetSubtargetInfo.h"#include "llvm/Config/llvm-config.h"#include "llvm/IR/Attributes.h"#include "llvm/IR/DebugLoc.h"#include "llvm/IR/Function.h"#include "llvm/MC/LaneBitmask.h"#include "llvm/MC/MCInstrDesc.h"#include "llvm/MC/MCInstrItineraries.h"#include "llvm/MC/MCRegisterInfo.h"#include "llvm/Pass.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Compiler.h"#include "llvm/Support/Debug.h"#include "llvm/Support/MathExtras.h"#include "llvm/Support/raw_ostream.h"#include <algorithm>#include <cassert>#include <climits>#include <cstdint>#include <deque>#include <functional>#include <iterator>#include <map>#include <memory>#include <tuple>#include <utility>#include <vector>Go to the source code of this file.
Namespaces | |
| llvm | |
| This class represents lattice values for constants. | |
Macros | |
| #define | DEBUG_TYPE "pipeliner" |
Functions | |
| STATISTIC (NumTrytoPipeline, "Number of loops that we attempt to pipeline") | |
| STATISTIC (NumPipelined, "Number of loops software pipelined") | |
| STATISTIC (NumNodeOrderIssues, "Number of node order issues found") | |
| cl::opt< bool > | llvm::SwpEnableCopyToPhi ("pipeliner-enable-copytophi", cl::ReallyHidden, cl::init(true), cl::ZeroOrMore, cl::desc("Enable CopyToPhi DAG Mutation")) |
| INITIALIZE_PASS_BEGIN (MachinePipeliner, DEBUG_TYPE, "Modulo Software Pipelining", false, false) INITIALIZE_PASS_END(MachinePipeliner | |
| static void | getPhiRegs (MachineInstr &Phi, MachineBasicBlock *Loop, unsigned &InitVal, unsigned &LoopVal) |
| Return the register values for the operands of a Phi instruction. More... | |
| static unsigned | getInitPhiReg (MachineInstr &Phi, MachineBasicBlock *LoopBB) |
| Return the Phi register value that comes from the incoming block. More... | |
| static unsigned | getLoopPhiReg (MachineInstr &Phi, MachineBasicBlock *LoopBB) |
| Return the Phi register value that comes the loop block. More... | |
| static bool | isSuccOrder (SUnit *SUa, SUnit *SUb) |
| Return true if SUb can be reached from SUa following the chain edges. More... | |
| static bool | isDependenceBarrier (MachineInstr &MI, AliasAnalysis *AA) |
| Return true if the instruction causes a chain between memory references before and after it. More... | |
| static void | getUnderlyingObjects (MachineInstr *MI, SmallVectorImpl< Value *> &Objs, const DataLayout &DL) |
| Return the underlying objects for the memory references of an instruction. More... | |
| static void | swapAntiDependences (std::vector< SUnit > &SUnits) |
| Swap all the anti dependences in the DAG. More... | |
| static bool | ignoreDependence (const SDep &D, bool isPred) |
| Return true for DAG nodes that we ignore when computing the cost functions. More... | |
| static bool | pred_L (SetVector< SUnit *> &NodeOrder, SmallSetVector< SUnit *, 8 > &Preds, const NodeSet *S=nullptr) |
| Compute the Pred_L(O) set, as defined in the paper. More... | |
| static bool | succ_L (SetVector< SUnit *> &NodeOrder, SmallSetVector< SUnit *, 8 > &Succs, const NodeSet *S=nullptr) |
| Compute the Succ_L(O) set, as defined in the paper. More... | |
| static bool | computePath (SUnit *Cur, SetVector< SUnit *> &Path, SetVector< SUnit *> &DestNodes, SetVector< SUnit *> &Exclude, SmallPtrSet< SUnit *, 8 > &Visited) |
| Return true if there is a path from the specified node to any of the nodes in DestNodes. More... | |
| template<class S1Ty , class S2Ty > | |
| static bool | isSubset (S1Ty &Set1, S2Ty &Set2) |
| Return true if Set1 is a subset of Set2. More... | |
| static void | computeLiveOuts (MachineFunction &MF, RegPressureTracker &RPTracker, NodeSet &NS) |
| Compute the live-out registers for the instructions in a node-set. More... | |
| static bool | isIntersect (SmallSetVector< SUnit *, 8 > &Set1, const NodeSet &Set2, SmallSetVector< SUnit *, 8 > &Result) |
| Return true if Set1 contains elements in Set2. More... | |
| static void | replaceRegUsesAfterLoop (unsigned FromReg, unsigned ToReg, MachineBasicBlock *MBB, MachineRegisterInfo &MRI, LiveIntervals &LIS) |
| Replace all uses of FromReg that appear outside the specified basic block with ToReg. More... | |
| static bool | hasUseAfterLoop (unsigned Reg, MachineBasicBlock *BB, MachineRegisterInfo &MRI) |
| Return true if the register has a use that occurs outside the specified loop. More... | |
| static void | removePhis (MachineBasicBlock *BB, MachineBasicBlock *Incoming) |
| Remove the incoming block from the Phis in a basic block. More... | |
| static SUnit * | multipleIterations (SUnit *SU, SwingSchedulerDAG *DAG) |
| If an instruction has a use that spans multiple iterations, then return true. More... | |
Variables | |
| static cl::opt< bool > | EnableSWP ("enable-pipeliner", cl::Hidden, cl::init(true), cl::ZeroOrMore, cl::desc("Enable Software Pipelining")) |
| A command line option to turn software pipelining on or off. More... | |
| static cl::opt< bool > | EnableSWPOptSize ("enable-pipeliner-opt-size", cl::desc("Enable SWP at Os."), cl::Hidden, cl::init(false)) |
| A command line option to enable SWP at -Os. More... | |
| static cl::opt< int > | SwpMaxMii ("pipeliner-max-mii", cl::desc("Size limit for the MII."), cl::Hidden, cl::init(27)) |
| A command line argument to limit minimum initial interval for pipelining. More... | |
| static cl::opt< int > | SwpMaxStages ("pipeliner-max-stages", cl::desc("Maximum stages allowed in the generated scheduled."), cl::Hidden, cl::init(3)) |
| A command line argument to limit the number of stages in the pipeline. More... | |
| static cl::opt< bool > | SwpPruneDeps ("pipeliner-prune-deps", cl::desc("Prune dependences between unrelated Phi nodes."), cl::Hidden, cl::init(true)) |
| A command line option to disable the pruning of chain dependences due to an unrelated Phi. More... | |
| static cl::opt< bool > | SwpPruneLoopCarried ("pipeliner-prune-loop-carried", cl::desc("Prune loop carried order dependences."), cl::Hidden, cl::init(true)) |
| A command line option to disable the pruning of loop carried order dependences. More... | |
| static cl::opt< int > | SwpLoopLimit ("pipeliner-max", cl::Hidden, cl::init(-1)) |
| static cl::opt< bool > | SwpIgnoreRecMII ("pipeliner-ignore-recmii", cl::ReallyHidden, cl::init(false), cl::ZeroOrMore, cl::desc("Ignore RecMII")) |
| DEBUG_TYPE | |
| Modulo Software | Pipelining |
| Modulo Software | false |
| #define DEBUG_TYPE "pipeliner" |
Definition at line 95 of file MachinePipeliner.cpp.
|
static |
Compute the live-out registers for the instructions in a node-set.
The live-out registers are those that are defined in the node-set, but not used. Except for use operands of Phis.
Definition at line 1327 of file MachinePipeliner.cpp.
References llvm::RegPressureTracker::addLiveRegs(), llvm::SetVector< T, Vector, Set >::begin(), llvm::SmallPtrSetImplBase::clear(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear(), llvm::NodeSet::clear(), llvm::RegPressureTracker::closeBottom(), llvm::NodeSet::compareRecMII(), computePath(), llvm::SmallSet< T, N, C >::count(), llvm::SetVector< T, Vector, Set >::count(), llvm::dbgs(), llvm::SetVector< T, Vector, Set >::empty(), llvm::NodeSet::empty(), llvm::SetVector< T, Vector, Set >::end(), llvm::RegPressureDelta::Excess, llvm::RegPressureTracker::getMaxUpwardPressureDelta(), llvm::LaneBitmask::getNone(), llvm::PressureChange::getPSet(), llvm::MachineFunction::getRegInfo(), llvm::TargetSubtargetInfo::getRegisterInfo(), llvm::MachineFunction::getSubtarget(), llvm::PressureChange::getUnitInc(), I, llvm::RegPressureTracker::init(), llvm::SetVector< T, Vector, Set >::insert(), llvm::SmallSet< T, N, C >::insert(), llvm::NodeSet::insert(), llvm::MachineRegisterInfo::isAllocatable(), llvm::MachineInstr::isPHI(), isSubset(), llvm::PressureChange::isValid(), llvm::MCRegisterInfo::DiffListIterator::isValid(), llvm::TargetRegisterInfo::isVirtualRegister(), LLVM_DEBUG, llvm::RegisterPressure::MaxSetPressure, llvm::MachinePipeliner::MF, MI, MRI, N, llvm::SUnit::NodeNum, llvm::MachineInstr::operands(), pred_L(), llvm::SUnit::Preds, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::RegPressureTracker::recede(), Reg, llvm::MachinePipeliner::RegClassInfo, llvm::NodeSet::setColocate(), llvm::RegPressureTracker::setPos(), SI, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size(), llvm::sort(), succ_L(), llvm::Successor, llvm::SUnit::Succs, and TRI.
|
static |
Return true if there is a path from the specified node to any of the nodes in DestNodes.
Keep track and return the nodes in any path.
Definition at line 1292 of file MachinePipeliner.cpp.
References llvm::SDep::Anti, llvm::SetVector< T, Vector, Set >::count(), llvm::SetVector< T, Vector, Set >::insert(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SUnit::isBoundaryNode(), llvm::SUnit::Preds, SI, and llvm::SUnit::Succs.
Referenced by computeLiveOuts().
|
static |
Return the Phi register value that comes from the incoming block.
Definition at line 430 of file MachinePipeliner.cpp.
References llvm::MachineOperand::getMBB(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), and llvm::MachineOperand::getReg().
Referenced by hasUseAfterLoop(), and removePhis().
|
static |
Return the Phi register value that comes the loop block.
Definition at line 438 of file MachinePipeliner.cpp.
References llvm::MachineOperand::getMBB(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), and llvm::MachineOperand::getReg().
Referenced by getUnderlyingObjects(), hasUseAfterLoop(), llvm::SMSchedule::isLoopCarriedDefOfUse(), and removePhis().
|
static |
Return the register values for the operands of a Phi instruction.
This function assume the instruction is a Phi.
Definition at line 414 of file MachinePipeliner.cpp.
References assert(), llvm::MachineOperand::getMBB(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), and llvm::MachineInstr::isPHI().
Referenced by hasUseAfterLoop(), llvm::SMSchedule::isLoopCarried(), llvm::SwingSchedulerDAG::isLoopCarriedDep(), and removePhis().
|
static |
Return the underlying objects for the memory references of an instruction.
This function calls the code in ValueTracking, but first checks that the instruction has a memory operand.
Definition at line 478 of file MachinePipeliner.cpp.
References llvm::SUnit::addPred(), llvm::TargetSubtargetInfo::adjustSchedDependency(), llvm::AAResults::alias(), llvm::SDep::Anti, llvm::TargetInstrInfo::areMemAccessesTriviallyDisjoint(), assert(), llvm::SDep::Barrier, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::InstrItineraryData::beginStage(), C, llvm::DFAPacketizer::canReserveResources(), llvm::MapVector< KeyT, ValueT, MapType, VectorType >::clear(), llvm::SmallVectorImpl< T >::clear(), llvm::countPopulation(), llvm::TargetInstrInfo::CreateTargetScheduleState(), D, llvm::SDep::Data, DefMI, E, llvm::SmallVectorBase::empty(), llvm::MapVector< KeyT, ValueT, MapType, VectorType >::end(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::InstrItineraryData::endStage(), F(), llvm::MapVector< KeyT, ValueT, MapType, VectorType >::find(), llvm::UndefValue::get(), llvm::MachineMemOperand::getAAInfo(), llvm::Function::getContext(), llvm::MachineFunction::getDataLayout(), llvm::MachineInstr::getDesc(), llvm::MachineBasicBlock::getFirstNonPHI(), llvm::MachineBasicBlock::getFirstTerminator(), llvm::MachineFunction::getFunction(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::TargetSubtargetInfo::getInstrItineraryData(), getLoopPhiReg(), llvm::TargetInstrInfo::getMemOperandWithOffset(), llvm::MachineMemOperand::getOffset(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineInstr::getParent(), llvm::MachineOperand::getReg(), llvm::MCInstrDesc::getSchedClass(), llvm::MachineFunction::getSubtarget(), llvm::GetUnderlyingObjects(), llvm::MachineMemOperand::getValue(), llvm::Type::getVoidTy(), llvm::MachineInstr::hasOneMemOperand(), I, llvm::ARM_PROC::IE, llvm::MachinePipeliner::InstrItins, isDependenceBarrier(), llvm::MachineOperand::isIdenticalTo(), llvm::isIdentifiedObject(), llvm::MachineInstr::isPHI(), isSuccOrder(), llvm::TargetInstrInfo::isZeroCost(), llvm::SPII::Load, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::MachineInstr::mayLoad(), llvm::MachineInstr::mayStore(), llvm::MachineInstr::memoperands_begin(), llvm::MachinePipeliner::MF, MI, MRI, llvm::NoAlias, llvm::SUnit::NodeNum, llvm::MachineInstr::operands_begin(), llvm::MachineInstr::operands_end(), llvm::SDep::Order, P, llvm::SUnit::Preds, llvm::SmallVectorTemplateBase< T >::push_back(), Reg, llvm::SUnit::removePred(), llvm::DFAPacketizer::reserveResources(), llvm::SDep::setLatency(), llvm::SmallVectorBase::size(), llvm::ARM_MB::ST, SwpPruneDeps, llvm::MachinePipeliner::TII, TRI, llvm::LocationSize::unknown(), and UseMI.
|
static |
Return true if the register has a use that occurs outside the specified loop.
Definition at line 2091 of file MachinePipeliner.cpp.
References llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::count(), llvm::tgtok::Def, E, llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::MachineBasicBlock::getFirstNonPHI(), getInitPhiReg(), getLoopPhiReg(), llvm::MachineInstr::getOperand(), llvm::MachineInstr::getParent(), getPhiRegs(), llvm::MachineOperand::getReg(), llvm::TargetSubtargetInfo::getRegisterInfo(), llvm::SMSchedule::getStagesForPhi(), llvm::SMSchedule::getStagesForReg(), llvm::MachineFunction::getSubtarget(), I, llvm::MachineBasicBlock::instr_begin(), llvm::MachineBasicBlock::instr_end(), llvm::MachineOperand::isDef(), llvm::SMSchedule::isLoopCarried(), llvm::MachineInstr::isPHI(), llvm::TargetRegisterInfo::isPhysicalRegister(), llvm::MachineOperand::isReg(), llvm::TargetRegisterInfo::isVirtualRegister(), llvm::make_range(), llvm::max(), llvm::MachinePipeliner::MF, MI, MRI, llvm::MachineBasicBlock::phis(), replaceRegUsesAfterLoop(), llvm::SMSchedule::stageScheduled(), llvm::MachinePipeliner::TII, TRI, llvm::MachineRegisterInfo::use_begin(), and llvm::MachineRegisterInfo::use_end().
Return true for DAG nodes that we ignore when computing the cost functions.
We ignore the back-edge recurrence in order to avoid unbounded recursion in the calculation of the ASAP, ALAP, etc functions.
Definition at line 1134 of file MachinePipeliner.cpp.
References llvm::SDep::Anti, llvm::dbgs(), E, llvm::SDep::getKind(), I, llvm::SDep::isArtificial(), LLVM_DEBUG, llvm::max(), pred, llvm::SUnit::Preds, and llvm::SUnit::Succs.
Referenced by isIntersect(), pred_L(), and succ_L().
| INITIALIZE_PASS_BEGIN | ( | MachinePipeliner | , |
| DEBUG_TYPE | , | ||
| "Modulo Software Pipelining" | , | ||
| false | , | ||
| false | |||
| ) |
|
static |
Return true if the instruction causes a chain between memory references before and after it.
Definition at line 469 of file MachinePipeliner.cpp.
References llvm::MachineInstr::hasOrderedMemoryRef(), llvm::MachineInstr::hasUnmodeledSideEffects(), llvm::MachineInstr::isCall(), llvm::MachineInstr::isDereferenceableInvariantLoad(), and llvm::MachineInstr::mayLoad().
Referenced by getUnderlyingObjects().
|
static |
Return true if Set1 contains elements in Set2.
The elements in common are returned in a different container.
Definition at line 1539 of file MachinePipeliner.cpp.
References llvm::MachineBasicBlock::addSuccessor(), llvm::TargetInstrInfo::analyzeBranch(), llvm::SDep::Anti, assert(), llvm::SmallVectorTemplateCommon< T >::back(), llvm::MachineBasicBlock::back(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::begin(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear(), llvm::MachineFunction::CloneMachineInstr(), llvm::NodeSet::compareRecMII(), llvm::SMSchedule::computeStart(), llvm::NodeSet::count(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::dbgs(), llvm::MachineBasicBlock::dump(), llvm::MachineInstr::dump(), E, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::empty(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::end(), llvm::SMSchedule::finalizeSchedule(), llvm::SMSchedule::getFinalCycle(), llvm::SMSchedule::getFirstCycle(), llvm::SUnit::getInstr(), llvm::SMSchedule::getInstructions(), llvm::MachineLoopInfo::getLoopFor(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::SMSchedule::getMaxStageCount(), llvm::MachineOperand::getMBB(), llvm::NodeSet::getNode(), llvm::NodeSet::getRecMII(), I, ignoreDependence(), llvm::tgtok::In, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert(), llvm::SMSchedule::insert(), llvm::MachineFunction::insert(), llvm::TargetInstrInfo::insertBranch(), llvm::MachineInstr::isPHI(), llvm::SMSchedule::isScheduledAtStage(), isSubset(), llvm::SMSchedule::isValidSchedule(), LLVM_DEBUG, llvm::max(), llvm::MachinePipeliner::MF, MI, llvm::MachinePipeliner::MLI, N, llvm::AArch64CC::NE, llvm::SUnit::NodeNum, llvm::NodeOrder, pred_L(), llvm::SUnit::Preds, llvm::MachineBasicBlock::push_back(), llvm::TargetInstrInfo::removeBranch(), llvm::MachineBasicBlock::replaceSuccessor(), llvm::SMSchedule::reset(), llvm::SMSchedule::setInitiationInterval(), llvm::MachineOperand::setMBB(), llvm::NodeSet::setRecMII(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size(), llvm::SMSchedule::stageScheduled(), llvm::MachineBasicBlock::succ_begin(), llvm::MachineBasicBlock::succ_end(), succ_L(), llvm::SUnit::Succs, SwpMaxStages, llvm::MachinePipeliner::TII, and llvm::MachineBasicBlock::transferSuccessors().
|
static |
Return true if Set1 is a subset of Set2.
Definition at line 1317 of file MachinePipeliner.cpp.
Referenced by computeLiveOuts(), and isIntersect().
Return true if SUb can be reached from SUa following the chain edges.
Definition at line 446 of file MachinePipeliner.cpp.
References llvm::SmallPtrSetImpl< PtrType >::count(), llvm::SmallVectorBase::empty(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SDep::Order, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T >::push_back(), SI, and llvm::SUnit::Succs.
Referenced by getUnderlyingObjects().
|
static |
If an instruction has a use that spans multiple iterations, then return true.
These instructions are characterized by having a back-ege to a Phi, which contains a reference to another Phi.
Definition at line 3216 of file MachinePipeliner.cpp.
References llvm::SDep::Data, llvm::SwingSchedulerDAG::isBackedge(), P, and llvm::SUnit::Preds.
|
static |
Compute the Pred_L(O) set, as defined in the paper.
The set is defined as the predecessors of the elements of NodeOrder that are not also in NodeOrder.
Definition at line 1228 of file MachinePipeliner.cpp.
References llvm::SDep::Anti, llvm::SetVector< T, Vector, Set >::begin(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear(), llvm::SetVector< T, Vector, Set >::count(), E, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::empty(), llvm::SetVector< T, Vector, Set >::end(), I, ignoreDependence(), and llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert().
Referenced by computeLiveOuts(), and isIntersect().
|
static |
Remove the incoming block from the Phis in a basic block.
Definition at line 2575 of file MachinePipeliner.cpp.
References llvm::MachineBasicBlock::addSuccessor(), llvm::TargetInstrInfo::areMemAccessesTriviallyDisjoint(), assert(), llvm::MachineBasicBlock::clear(), llvm::MachineFunction::CloneMachineInstr(), llvm::computeDelta(), llvm::count(), llvm::SMSchedule::cycleScheduled(), D, llvm::tgtok::Def, llvm::MachineFunction::DeleteMachineInstr(), E, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::MachineBasicBlock::eraseFromParent(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::TargetInstrInfo::getBaseAndOffsetPosition(), llvm::MachineOperand::getImm(), llvm::TargetInstrInfo::getIncrementValue(), getInitPhiReg(), getLoopPhiReg(), llvm::MachineFunction::getMachineMemOperand(), llvm::SMSchedule::getMaxStageCount(), llvm::MachineOperand::getMBB(), llvm::TargetInstrInfo::getMemOperandWithOffset(), llvm::MachineInstr::getMF(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getParent(), llvm::MachineInstr::getParent(), getPhiRegs(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::TargetSubtargetInfo::getRegisterInfo(), llvm::SMSchedule::getStagesForPhi(), llvm::MachineFunction::getSubtarget(), llvm::MachineRegisterInfo::getVRegDef(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetInstrInfo::insertBranch(), llvm::MachineBasicBlock::instr_rbegin(), llvm::MachineBasicBlock::instr_rend(), llvm::MachineOperand::isDef(), llvm::MachineInstr::isInlineAsm(), llvm::SMSchedule::isLoopCarried(), llvm::MachineInstr::isPHI(), llvm::TargetInstrInfo::isPostIncrement(), llvm::MachineOperand::isReg(), llvm::MachineInstr::isRegTiedToUseOperand(), llvm::MachineOperand::isUse(), llvm::TargetRegisterInfo::isVirtualRegister(), llvm::MachineInstr::memoperands(), llvm::MachineInstr::memoperands_empty(), llvm::MachinePipeliner::MF, MI, MRI, llvm::MachineBasicBlock::phis(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::TargetInstrInfo::reduceLoopCount(), Reg, llvm::MachineBasicBlock::removeSuccessor(), replaceRegUsesAfterLoop(), llvm::MachineOperand::setImm(), llvm::MachineInstr::setMemRefs(), llvm::MachineOperand::setReg(), llvm::SmallVectorBase::size(), llvm::SMSchedule::stageScheduled(), llvm::MachineInstr::tieOperands(), llvm::MachinePipeliner::TII, TRI, llvm::MemoryLocation::UnknownSize, and UseMI.
|
static |
Replace all uses of FromReg that appear outside the specified basic block with ToReg.
Definition at line 2073 of file MachinePipeliner.cpp.
References llvm::LiveIntervals::createEmptyInterval(), E, llvm::MachineOperand::getParent(), llvm::MachineInstr::getParent(), llvm::LiveIntervals::hasInterval(), I, llvm::RISCVFenceField::O, llvm::MachineOperand::setReg(), llvm::MachineRegisterInfo::use_begin(), and llvm::MachineRegisterInfo::use_end().
Referenced by hasUseAfterLoop(), and removePhis().
| STATISTIC | ( | NumTrytoPipeline | , |
| "Number of loops that we attempt to pipeline" | |||
| ) |
| STATISTIC | ( | NumPipelined | , |
| "Number of loops software pipelined" | |||
| ) |
| STATISTIC | ( | NumNodeOrderIssues | , |
| "Number of node order issues found" | |||
| ) |
|
static |
Compute the Succ_L(O) set, as defined in the paper.
The set is defined as the successors of the elements of NodeOrder that are not also in NodeOrder.
Definition at line 1261 of file MachinePipeliner.cpp.
References llvm::SDep::Anti, llvm::SetVector< T, Vector, Set >::begin(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear(), llvm::SetVector< T, Vector, Set >::count(), E, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::empty(), llvm::SetVector< T, Vector, Set >::end(), I, ignoreDependence(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert(), and SI.
Referenced by computeLiveOuts(), and isIntersect().
|
static |
Swap all the anti dependences in the DAG.
That means it is no longer a DAG, but we do this to find the circuits, and then change them back.
Definition at line 896 of file MachinePipeliner.cpp.
References llvm::SUnit::addPred(), llvm::ScheduleDAGTopologicalSort::AddPred(), llvm::SDep::Anti, llvm::cl::apply(), llvm::SDep::Artificial, assert(), llvm::SmallVectorTemplateCommon< T >::begin(), llvm::SmallPtrSetImpl< PtrType >::begin(), llvm::count(), D, llvm::SDep::Data, E, llvm::SmallPtrSetImplBase::empty(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::SmallPtrSetImpl< PtrType >::end(), llvm::SmallPtrSetImpl< PtrType >::erase(), F(), llvm::SUnit::getInstr(), llvm::SDep::getLatency(), llvm::SDep::getReg(), llvm::SDep::getSUnit(), I, llvm::MachineBasicBlock::insert(), llvm::MachineInstr::isCopy(), llvm::SwingSchedulerDAG::isLoopCarriedDep(), llvm::MachineInstr::isPHI(), llvm::ScheduleDAGTopologicalSort::IsReachable(), llvm::MachineInstr::isRegSequence(), N, llvm::SUnit::NumPreds, llvm::SDep::Order, llvm::SDep::Output, llvm::SUnit::Preds, llvm::SmallVectorTemplateBase< T >::push_back(), Reg, llvm::SUnit::removePred(), llvm::SDep::setLatency(), SI, llvm::SmallVectorBase::size(), llvm::ScheduleDAG::SUnits, true, and llvm::RISCVFenceField::W.
| DEBUG_TYPE |
Definition at line 167 of file MachinePipeliner.cpp.
|
static |
A command line option to turn software pipelining on or off.
Referenced by llvm::MachinePipeliner::runOnMachineFunction().
|
static |
A command line option to enable SWP at -Os.
Referenced by llvm::MachinePipeliner::runOnMachineFunction().
| Modulo Software false |
Definition at line 167 of file MachinePipeliner.cpp.
| Modulo Software Pipelining |
Definition at line 167 of file MachinePipeliner.cpp.
|
static |
Referenced by llvm::SwingSchedulerDAG::schedule().
|
static |
Referenced by llvm::MachinePipeliner::runOnMachineFunction().
|
static |
A command line argument to limit minimum initial interval for pipelining.
Referenced by llvm::SwingSchedulerDAG::schedule().
|
static |
A command line argument to limit the number of stages in the pipeline.
Referenced by isIntersect(), and llvm::SwingSchedulerDAG::schedule().
|
static |
A command line option to disable the pruning of chain dependences due to an unrelated Phi.
Referenced by getUnderlyingObjects().
|
static |
A command line option to disable the pruning of loop carried order dependences.
Referenced by llvm::SwingSchedulerDAG::isLoopCarriedDep().
1.8.13