#include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/None.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DomTreeUpdater.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <set>
#include <string>
#include <utility>
#include <vector>

Classes
struct	false::GCPtrLivenessData

struct	false::PartiallyConstructedSafepointRecord

Namespaces
	false

Macros
#define	DEBUG_TYPE "rewrite-statepoints-for-gc"

Typedefs
using	false::DefiningValueMapTy = MapVector< Value , Value >

using	false::StatepointLiveSetTy = SetVector< Value * >

using	false::RematerializedValueMapTy = MapVector< AssertingVH< Instruction >, AssertingVH< Value > >

Functions
static void	stripNonValidData (Module &M)
	The IR fed into RewriteStatepointsForGC may have had attributes and metadata implying dereferenceability that are no longer valid/correct after RewriteStatepointsForGC has run. More...

static bool	shouldRewriteStatepointsIn (Function &F)
	Returns true if this function should be rewritten by this pass. More...

	INITIALIZE_PASS_BEGIN (RewriteStatepointsForGCLegacyPass, "rewrite-statepoints-for-gc", "Make relocations explicit at statepoints", false, false) INITIALIZE_PASS_END(RewriteStatepointsForGCLegacyPass

static ArrayRef< Use >	GetDeoptBundleOperands (ImmutableCallSite CS)

static void	computeLiveInValues (DominatorTree &DT, Function &F, GCPtrLivenessData &Data)
	Compute the live-in set for every basic block in the function. More...

static void	findLiveSetAtInst (Instruction *inst, GCPtrLivenessData &Data, StatepointLiveSetTy &out)
	Given results from the dataflow liveness computation, find the set of live Values at a particular instruction. More...

static bool	isGCPointerType (Type *T)

static bool	isHandledGCPointerType (Type *T)

static bool	containsGCPtrType (Type *Ty)
	Returns true if this type contains a gc pointer whether we know how to handle that type or not. More...

static bool	isUnhandledGCPointerType (Type *Ty)

static std::string	suffixed_name_or (Value *V, StringRef Suffix, StringRef DefaultName)

static void	analyzeParsePointLiveness (DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallSite CS, PartiallyConstructedSafepointRecord &Result)

static bool	isKnownBaseResult (Value *V)
	Given the result of a call to findBaseDefiningValue, or findBaseOrBDV, is it known to be a base pointer? Or do we need to continue searching. More...

static BaseDefiningValueResult	findBaseDefiningValue (Value *I)
	Helper function for findBasePointer - Will return a value which either a) defines the base pointer for the input, b) blocks the simple search (i.e. More...

static BaseDefiningValueResult	findBaseDefiningValueOfVector (Value *I)
	Return a base defining value for the 'Index' element of the given vector instruction 'I'. More...

static Value *	findBaseDefiningValueCached (Value *I, DefiningValueMapTy &Cache)
	Returns the base defining value for this value. More...

static Value *	findBaseOrBDV (Value *I, DefiningValueMapTy &Cache)
	Return a base pointer for this value if known. More...

static raw_ostream &	operator<< (raw_ostream &OS, const BDVState &State)

static BDVState	meetBDVStateImpl (const BDVState &LHS, const BDVState &RHS)

static BDVState	meetBDVState (const BDVState &LHS, const BDVState &RHS)

static Value *	findBasePointer (Value *I, DefiningValueMapTy &Cache)
	For a given value or instruction, figure out what base ptr its derived from. More...

static void	findBasePointers (const StatepointLiveSetTy &live, MapVector< Value , Value > &PointerToBase, DominatorTree *DT, DefiningValueMapTy &DVCache)

static void	findBasePointers (DominatorTree &DT, DefiningValueMapTy &DVCache, CallSite CS, PartiallyConstructedSafepointRecord &result)
	Find the required based pointers (and adjust the live set) for the given parse point. More...

static void	recomputeLiveInValues (GCPtrLivenessData &RevisedLivenessData, CallSite CS, PartiallyConstructedSafepointRecord &result)
	Given an updated version of the dataflow liveness results, update the liveset and base pointer maps for the call site CS. More...

static void	recomputeLiveInValues (Function &F, DominatorTree &DT, ArrayRef< CallSite > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records)

static BasicBlock *	normalizeForInvokeSafepoint (BasicBlock BB, BasicBlock InvokeParent, DominatorTree &DT)

static AttributeList	legalizeCallAttributes (AttributeList AL)

static void	CreateGCRelocates (ArrayRef< Value > LiveVariables, const int LiveStart, ArrayRef< Value > BasePtrs, Instruction *StatepointToken, IRBuilder<> Builder)
	Helper function to place all gc relocates necessary for the given statepoint. More...

static StringRef	getDeoptLowering (CallSite CS)

static void	makeStatepointExplicitImpl (const CallSite CS, const SmallVectorImpl< Value > &BasePtrs, const SmallVectorImpl< Value > &LiveVariables, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements)

static void	makeStatepointExplicit (DominatorTree &DT, CallSite CS, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements)

static void	insertRelocationStores (iterator_range< Value::user_iterator > GCRelocs, DenseMap< Value , Value > &AllocaMap, DenseSet< Value *> &VisitedLiveValues)

static void	insertRematerializationStores (const RematerializedValueMapTy &RematerializedValues, DenseMap< Value , Value > &AllocaMap, DenseSet< Value *> &VisitedLiveValues)

static void	relocationViaAlloca (Function &F, DominatorTree &DT, ArrayRef< Value *> Live, ArrayRef< PartiallyConstructedSafepointRecord > Records)
	Do all the relocation update via allocas and mem2reg. More...

template<typename T >
static void	unique_unsorted (SmallVectorImpl< T > &Vec)
	Implement a unique function which doesn't require we sort the input vector. More...

static void	insertUseHolderAfter (CallSite &CS, const ArrayRef< Value > Values, SmallVectorImpl< CallInst > &Holders)
	Insert holders so that each Value is obviously live through the entire lifetime of the call. More...

static void	findLiveReferences (Function &F, DominatorTree &DT, ArrayRef< CallSite > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records)

static Value *	findRematerializableChainToBasePointer (SmallVectorImpl< Instruction > &ChainToBase, Value CurrentValue)

static unsigned	chainToBasePointerCost (SmallVectorImpl< Instruction *> &Chain, TargetTransformInfo &TTI)

static bool	AreEquivalentPhiNodes (PHINode &OrigRootPhi, PHINode &AlternateRootPhi)

static void	rematerializeLiveValues (CallSite CS, PartiallyConstructedSafepointRecord &Info, TargetTransformInfo &TTI)

static bool	insertParsePoints (Function &F, DominatorTree &DT, TargetTransformInfo &TTI, SmallVectorImpl< CallSite > &ToUpdate)

template<typename AttrHolder >
static void	RemoveNonValidAttrAtIndex (LLVMContext &Ctx, AttrHolder &AH, unsigned Index)

static void	stripNonValidAttributesFromPrototype (Function &F)

static void	stripInvalidMetadataFromInstruction (Instruction &I)
	Certain metadata on instructions are invalid after running RS4GC. More...

static void	stripNonValidDataFromBody (Function &F)

static void	computeLiveInValues (BasicBlock::reverse_iterator Begin, BasicBlock::reverse_iterator End, SetVector< Value *> &LiveTmp)
	Compute the live-in set for the location rbegin starting from the live-out set of the basic block. More...

static void	computeLiveOutSeed (BasicBlock BB, SetVector< Value > &LiveTmp)

static SetVector< Value * >	computeKillSet (BasicBlock *BB)

static void	checkBasicSSA (DominatorTree &DT, SetVector< Value > &Live, Instruction TI, bool TermOkay=false)
	Check that the items in 'Live' dominate 'TI'. More...

static void	checkBasicSSA (DominatorTree &DT, GCPtrLivenessData &Data, BasicBlock &BB)
	Check that all the liveness sets used during the computation of liveness obey basic SSA properties. More...

Variables
static cl::opt< bool >	PrintLiveSet ("spp-print-liveset", cl::Hidden, cl::init(false))

static cl::opt< bool >	PrintLiveSetSize ("spp-print-liveset-size", cl::Hidden, cl::init(false))

static cl::opt< bool >	PrintBasePointers ("spp-print-base-pointers", cl::Hidden, cl::init(false))

static cl::opt< unsigned >	RematerializationThreshold ("spp-rematerialization-threshold", cl::Hidden, cl::init(6))

static bool	ClobberNonLive = false

static cl::opt< bool, true >	ClobberNonLiveOverride ("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden)

static cl::opt< bool >	AllowStatepointWithNoDeoptInfo ("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true))

rewrite statepoints for	gc

rewrite statepoints for Make relocations at	statepoints

rewrite statepoints for Make relocations at	false

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "rewrite-statepoints-for-gc"

Definition at line 80 of file RewriteStatepointsForGC.cpp.

Function Documentation

◆ analyzeParsePointLiveness()

static void analyzeParsePointLiveness	(	DominatorTree &	DT,
		GCPtrLivenessData &	OriginalLivenessData,
		CallSite	CS,
		PartiallyConstructedSafepointRecord &	Result
	)

static

Definition at line 374 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::dbgs(), findBaseDefiningValue(), findLiveSetAtInst(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getCalledValue(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getInstruction(), llvm::Value::getName(), I, isKnownBaseResult(), PrintLiveSet, and PrintLiveSetSize.

Referenced by findLiveReferences().

◆ AreEquivalentPhiNodes()

static bool AreEquivalentPhiNodes	(	PHINode &	OrigRootPhi,
		PHINode &	AlternateRootPhi
	)

static

Definition at line 1995 of file RewriteStatepointsForGC.cpp.

References llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, InlineBuckets, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::PHINode::getNumIncomingValues(), and llvm::Instruction::getParent().

Referenced by rematerializeLiveValues().

◆ chainToBasePointerCost()

static unsigned chainToBasePointerCost	(	SmallVectorImpl< Instruction *> &	Chain,
		TargetTransformInfo &	TTI
	)

static

Definition at line 1964 of file RewriteStatepointsForGC.cpp.

References assert(), GEP, llvm::TargetTransformInfo::getAddressComputationCost(), llvm::TargetTransformInfo::getCastInstrCost(), and llvm_unreachable.

Referenced by rematerializeLiveValues().

◆ checkBasicSSA() [1/2]

static void checkBasicSSA	(	DominatorTree &	DT,
		SetVector< Value *> &	Live,
		Instruction *	TI,
		bool	TermOkay = `false`
	)

static

Check that the items in 'Live' dominate 'TI'.

This is used as a basic sanity check for the liveness computation.

Definition at line 2677 of file RewriteStatepointsForGC.cpp.

References assert(), and llvm::DominatorTree::dominates().

Referenced by checkBasicSSA(), and computeLiveInValues().

◆ checkBasicSSA() [2/2]

static void checkBasicSSA	(	DominatorTree &	DT,
		GCPtrLivenessData &	Data,
		BasicBlock &	BB
	)

static

Check that all the liveness sets used during the computation of liveness obey basic SSA properties.

This is useful for finding cases where we miss a def.

Definition at line 2695 of file RewriteStatepointsForGC.cpp.

References checkBasicSSA(), and llvm::BasicBlock::getTerminator().

◆ computeKillSet()

static SetVector<Value *> computeKillSet ( BasicBlock * BB )

static

Definition at line 2666 of file RewriteStatepointsForGC.cpp.

References llvm::Value::getType(), llvm::SetVector< T, Vector, Set >::insert(), and isHandledGCPointerType().

Referenced by computeLiveInValues().

◆ computeLiveInValues() [1/2]

static void computeLiveInValues	(	DominatorTree &	DT,
		Function &	F,
		GCPtrLivenessData &	Data
	)

static

Compute the live-in set for every basic block in the function.

Definition at line 2703 of file RewriteStatepointsForGC.cpp.

References assert(), checkBasicSSA(), computeKillSet(), computeLiveOutSeed(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::empty(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert(), llvm::RegState::Kill, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::pop_back_val(), llvm::pred_begin(), llvm::pred_end(), llvm::SetVector< T, Vector, Set >::set_subtract(), llvm::SetVector< T, Vector, Set >::set_union(), llvm::SetVector< T, Vector, Set >::size(), and llvm::successors().

Referenced by findLiveReferences(), findLiveSetAtInst(), GetDeoptBundleOperands(), and recomputeLiveInValues().

◆ computeLiveInValues() [2/2]

static void computeLiveInValues	(	BasicBlock::reverse_iterator	Begin,
		BasicBlock::reverse_iterator	End,
		SetVector< Value *> &	LiveTmp
	)

static

Compute the live-in set for the location rbegin starting from the live-out set of the basic block.

Definition at line 2617 of file RewriteStatepointsForGC.cpp.

◆ computeLiveOutSeed()

static void computeLiveOutSeed	(	BasicBlock *	BB,
		SetVector< Value *> &	LiveTmp
	)

static

Definition at line 2650 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::dyn_cast(), llvm::PHINode::getIncomingValueForBlock(), llvm::Value::getType(), I, llvm::SetVector< T, Vector, Set >::insert(), isHandledGCPointerType(), isUnhandledGCPointerType(), and llvm::successors().

Referenced by computeLiveInValues().

◆ containsGCPtrType()

static bool containsGCPtrType ( Type * Ty )

static

Returns true if this type contains a gc pointer whether we know how to handle that type or not.

Definition at line 342 of file RewriteStatepointsForGC.cpp.

References llvm::any_of(), isGCPointerType(), and llvm::ARM_MB::ST.

Referenced by isUnhandledGCPointerType().

◆ CreateGCRelocates()

static void CreateGCRelocates	(	ArrayRef< Value *>	LiveVariables,
		const int	LiveStart,
		ArrayRef< Value *>	BasePtrs,
		Instruction *	StatepointToken,
		IRBuilder<>	Builder
	)

static

Helper function to place all gc relocates necessary for the given statepoint.

Inputs: liveVariables - list of variables to be relocated. liveStart - index of the first live variable. basePtrs - base pointers. statepointToken - statepoint instruction to which relocates should be bound. Builder - Llvm IR builder to be used to construct new calls.

Definition at line 1273 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::ArrayRef< T >::begin(), llvm::CallingConv::Cold, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::IRBuilder< T, Inserter >::CreateCall(), D, llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::end(), llvm::Instruction::eraseFromParent(), llvm::Intrinsic::experimental_deoptimize, llvm::Intrinsic::experimental_gc_relocate, llvm::find(), llvm::VectorType::get(), getCalledFunction(), llvm::Module::getContext(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::getInt32(), llvm::Type::getInt8PtrTy(), llvm::Function::getIntrinsicID(), llvm::Instruction::getModule(), llvm::Instruction::getParent(), llvm::BasicBlock::getTerminator(), isHandledGCPointerType(), llvm::Value::replaceAllUsesWith(), llvm::CallBase::setCallingConv(), llvm::ArrayRef< T >::size(), and suffixed_name_or().

Referenced by makeStatepointExplicitImpl().

◆ findBaseDefiningValue()

static BaseDefiningValueResult findBaseDefiningValue ( Value * I )

static

Helper function for findBasePointer - Will return a value which either a) defines the base pointer for the input, b) blocks the simple search (i.e.

a PHI or Select of two derived pointers), or c) involves a change from pointer to vector type or back.

Definition at line 497 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::tgtok::Def, llvm::Intrinsic::experimental_gc_relocate, llvm::Intrinsic::experimental_gc_statepoint, findBaseDefiningValueOfVector(), llvm::Intrinsic::gcroot, GEP, llvm::ConstantPointerNull::get(), llvm::Value::getType(), llvm::Type::isPtrOrPtrVectorTy(), llvm::Type::isVectorTy(), and llvm_unreachable.

Referenced by analyzeParsePointLiveness(), findBaseDefiningValueCached(), and findBaseDefiningValueOfVector().

◆ findBaseDefiningValueCached()

static Value* findBaseDefiningValueCached	(	Value *	I,
		DefiningValueMapTy &	Cache
	)

static

Returns the base defining value for this value.

Definition at line 616 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::dbgs(), findBaseDefiningValue(), llvm::Value::getName(), I, and LLVM_DEBUG.

Referenced by findBaseOrBDV().

◆ findBaseDefiningValueOfVector()

static BaseDefiningValueResult findBaseDefiningValueOfVector ( Value * I )

static

Return a base defining value for the 'Index' element of the given vector instruction 'I'.

If Index is null, returns a BDV for the entire vector 'I'. As an optimization, this method will try to determine when the element is known to already be a base pointer. If this can be established, the second value in the returned pair will be true. Note that either a vector or a pointer typed value can be returned. For the former, the vector returned is a BDV (and possibly a base) of the entire vector 'I'. If the later, the return pointer is a BDV (or possibly a base) for the particular element in 'I'.

Definition at line 439 of file RewriteStatepointsForGC.cpp.

References assert(), findBaseDefiningValue(), GEP, llvm::ConstantAggregateZero::get(), llvm::Value::getType(), and I.

Referenced by findBaseDefiningValue().

◆ findBaseOrBDV()

static Value* findBaseOrBDV	(	Value *	I,
		DefiningValueMapTy &	Cache
	)

static

Return a base pointer for this value if known.

Otherwise, return it's base defining value.

Definition at line 629 of file RewriteStatepointsForGC.cpp.

References llvm::tgtok::Def, and findBaseDefiningValueCached().

Referenced by findBasePointer().

◆ findBasePointer()

static Value* findBasePointer	(	Value *	I,
		DefiningValueMapTy &	Cache
	)

static

For a given value or instruction, figure out what base ptr its derived from.

For gc objects, this is simply itself. On success, returns a value which is the base pointer. (This is reliable and can be used for relocation.) On failure, returns nullptr.

Create and insert a new instruction which will represent the base of the given instruction 'I'.

Definition at line 766 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::SelectInst::Create(), llvm::ExtractElementInst::Create(), llvm::InsertElementInst::Create(), llvm::PHINode::Create(), llvm::dbgs(), llvm::tgtok::Def, llvm::SmallVectorBase::empty(), findBaseOrBDV(), llvm::MDNode::get(), llvm::UndefValue::get(), llvm::Value::getContext(), llvm::SelectInst::getFalseValue(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::Value::getName(), llvm::PHINode::getNumIncomingValues(), llvm::Instruction::getParent(), llvm::BasicBlock::getTerminator(), llvm::SelectInst::getTrueValue(), llvm::Value::getType(), I, llvm::ARM_PROC::IE, llvm::MapVector< KeyT, ValueT, MapType, VectorType >::insert(), isKnownBaseResult(), LLVM_DEBUG, llvm_unreachable, meetBDVState(), Name, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::pred_size(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::Instruction::setMetadata(), SI, llvm::Value::stripPointerCasts(), suffixed_name_or(), and llvm::RegState::Undef.

Referenced by findBasePointers().

◆ findBasePointers() [1/2]

static void findBasePointers	(	const StatepointLiveSetTy &	live,
		MapVector< Value , Value > &	PointerToBase,
		DominatorTree *	DT,
		DefiningValueMapTy &	DVCache
	)

static

Definition at line 1164 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::DominatorTree::dominates(), findBasePointer(), and getParent().

Referenced by findBasePointers(), and insertParsePoints().

◆ findBasePointers() [2/2]

static void findBasePointers	(	DominatorTree &	DT,
		DefiningValueMapTy &	DVCache,
		CallSite	CS,
		PartiallyConstructedSafepointRecord &	result
	)

static

Find the required based pointers (and adjust the live set) for the given parse point.

Definition at line 1180 of file RewriteStatepointsForGC.cpp.

References llvm::errs(), findBasePointers(), PrintBasePointers, and recomputeLiveInValues().

◆ findLiveReferences()

static void findLiveReferences	(	Function &	F,
		DominatorTree &	DT,
		ArrayRef< CallSite >	toUpdate,
		MutableArrayRef< struct PartiallyConstructedSafepointRecord >	records
	)

static

Definition at line 1921 of file RewriteStatepointsForGC.cpp.

References analyzeParsePointLiveness(), computeLiveInValues(), info, and llvm::ArrayRef< T >::size().

Referenced by insertParsePoints().

◆ findLiveSetAtInst()

static void findLiveSetAtInst	(	Instruction *	inst,
		GCPtrLivenessData &	Data,
		StatepointLiveSetTy &	out
	)

static

Given results from the dataflow liveness computation, find the set of live Values at a particular instruction.

Definition at line 2770 of file RewriteStatepointsForGC.cpp.

References assert(), computeLiveInValues(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::Instruction::getParent(), llvm::ilist_iterator< OptionsT, IsReverse, IsConst >::getReverse(), and llvm::BasicBlock::rbegin().

Referenced by analyzeParsePointLiveness(), GetDeoptBundleOperands(), and recomputeLiveInValues().

◆ findRematerializableChainToBasePointer()

static Value* findRematerializableChainToBasePointer	(	SmallVectorImpl< Instruction *> &	ChainToBase,
		Value *	CurrentValue
	)

static

Definition at line 1938 of file RewriteStatepointsForGC.cpp.

References GEP, and llvm::SmallVectorTemplateBase< T >::push_back().

Referenced by rematerializeLiveValues().

◆ GetDeoptBundleOperands()

static ArrayRef<Use> GetDeoptBundleOperands ( ImmutableCallSite CS )

static

Definition at line 289 of file RewriteStatepointsForGC.cpp.

References AllowStatepointWithNoDeoptInfo, assert(), computeLiveInValues(), llvm::Data, F(), findLiveSetAtInst(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getOperandBundle(), llvm::Optional< T >::getValue(), llvm::Optional< T >::hasValue(), llvm::OperandBundleUse::Inputs, llvm::None, and llvm::LLVMContext::OB_deopt.

Referenced by insertParsePoints(), and makeStatepointExplicitImpl().

◆ getDeoptLowering()

static StringRef getDeoptLowering ( CallSite CS )

static

Definition at line 1402 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::AttributeList::FunctionIndex, llvm::AttributeList::getAttribute(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getAttributes(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getCalledFunction(), llvm::Function::getFnAttribute(), llvm::Attribute::getValueAsString(), llvm::AttributeList::hasAttribute(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::hasFnAttr(), and llvm::Function::hasFnAttribute().

Referenced by makeStatepointExplicitImpl().

◆ INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	RewriteStatepointsForGCLegacyPass	,
		"rewrite-statepoints-for-gc"	,
		"Make relocations explicit at statepoints"	,
		false	,
		false
	)

Referenced by llvm::createRewriteStatepointsForGCLegacyPass().

◆ insertParsePoints()

static bool insertParsePoints	(	Function &	F,
		DominatorTree &	DT,
		TargetTransformInfo &	TTI,
		SmallVectorImpl< CallSite > &	ToUpdate
	)

static

Definition at line 2177 of file RewriteStatepointsForGC.cpp.

References Arg, assert(), llvm::SmallVectorTemplateCommon< T >::begin(), llvm::DominatorTree::dominates(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::errs(), F(), findBasePointers(), findLiveReferences(), llvm::StatepointBase< FunTy, InstructionTy, ValueTy, CallSiteTy >::gc_args(), llvm::StatepointBase< FunTy, InstructionTy, ValueTy, CallSiteTy >::gc_args_begin(), llvm::StatepointBase< FunTy, InstructionTy, ValueTy, CallSiteTy >::gc_args_end(), GetDeoptBundleOperands(), llvm::Instruction::getFunction(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getInstruction(), llvm::Value::getType(), Info, info, llvm::SmallVectorImpl< T >::insert(), insertUseHolderAfter(), isHandledGCPointerType(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::isInvoke(), llvm::DominatorTree::isReachableFromEntry(), isUnhandledGCPointerType(), makeStatepointExplicit(), normalizeForInvokeSafepoint(), PrintBasePointers, llvm::SmallVectorTemplateBase< T >::push_back(), recomputeLiveInValues(), relocationViaAlloca(), rematerializeLiveValues(), llvm::SmallVectorBase::size(), and unique_unsorted().

Referenced by llvm::RewriteStatepointsForGC::runOnFunction().

◆ insertRelocationStores()

static void insertRelocationStores	(	iterator_range< Value::user_iterator >	GCRelocs,
		DenseMap< Value , Value > &	AllocaMap,
		DenseSet< Value *> &	VisitedLiveValues
	)

static

Definition at line 1638 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::dyn_cast(), llvm::GCRelocateInst::getDerivedPtr(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getNextNode(), llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), llvm::Instruction::insertAfter(), llvm::SPII::Store, and suffixed_name_or().

Referenced by relocationViaAlloca().

◆ insertRematerializationStores()

static void insertRematerializationStores	(	const RematerializedValueMapTy &	RematerializedValues,
		DenseMap< Value , Value > &	AllocaMap,
		DenseSet< Value *> &	VisitedLiveValues
	)

static

Definition at line 1672 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), llvm::Instruction::insertAfter(), and llvm::SPII::Store.

Referenced by relocationViaAlloca().

◆ insertUseHolderAfter()

static void insertUseHolderAfter	(	CallSite &	CS,
		const ArrayRef< Value *>	Values,
		SmallVectorImpl< CallInst *> &	Holders
	)

static

Insert holders so that each Value is obviously live through the entire lifetime of the call.

Definition at line 1896 of file RewriteStatepointsForGC.cpp.

References llvm::CallInst::Create(), llvm::ArrayRef< T >::empty(), llvm::FunctionType::get(), llvm::Module::getContext(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getInstruction(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::Instruction::getModule(), llvm::Module::getOrInsertFunction(), llvm::Type::getVoidTy(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::isCall(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by insertParsePoints().

◆ isGCPointerType()

static bool isGCPointerType ( Type * T )

static

Definition at line 314 of file RewriteStatepointsForGC.cpp.

Referenced by containsGCPtrType(), and isHandledGCPointerType().

◆ isHandledGCPointerType()

static bool isHandledGCPointerType ( Type * T )

static

Definition at line 327 of file RewriteStatepointsForGC.cpp.

References isGCPointerType().

Referenced by computeKillSet(), computeLiveOutSeed(), CreateGCRelocates(), insertParsePoints(), and isUnhandledGCPointerType().

◆ isKnownBaseResult()

static bool isKnownBaseResult ( Value * V )

static

Given the result of a call to findBaseDefiningValue, or findBaseOrBDV, is it known to be a base pointer? Or do we need to continue searching.

Definition at line 642 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::dbgs(), llvm::dump(), LLVM_DUMP_METHOD, llvm::operator!=(), llvm::operator==(), Other, print(), and llvm::Unknown.

Referenced by analyzeParsePointLiveness(), findBasePointer(), and recomputeLiveInValues().

◆ isUnhandledGCPointerType()

static bool isUnhandledGCPointerType ( Type * Ty )

static

Definition at line 357 of file RewriteStatepointsForGC.cpp.

References containsGCPtrType(), and isHandledGCPointerType().

Referenced by computeLiveOutSeed(), and insertParsePoints().

◆ legalizeCallAttributes()

static AttributeList legalizeCallAttributes ( AttributeList AL )

static

Definition at line 1245 of file RewriteStatepointsForGC.cpp.

References A, llvm::AttributeList::FunctionIndex, llvm::AttributeSet::get(), llvm::AttributeList::get(), llvm::AttributeList::getContext(), llvm::AttributeList::getFnAttributes(), llvm::AttributeList::isEmpty(), llvm::isStatepointDirectiveAttr(), llvm::Attribute::ReadNone, llvm::Attribute::ReadOnly, llvm::AttrBuilder::remove(), and llvm::AttrBuilder::removeAttribute().

Referenced by makeStatepointExplicitImpl().

◆ makeStatepointExplicit()

static void makeStatepointExplicit	(	DominatorTree &	DT,
		CallSite	CS,
		PartiallyConstructedSafepointRecord &	Result,
		std::vector< DeferredReplacement > &	Replacements
	)

static

Definition at line 1609 of file RewriteStatepointsForGC.cpp.

References assert(), makeStatepointExplicitImpl(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorImpl< T >::reserve(), and llvm::SmallVectorBase::size().

Referenced by insertParsePoints().

◆ makeStatepointExplicitImpl()

static void makeStatepointExplicitImpl	(	const CallSite	CS,
		const SmallVectorImpl< Value *> &	BasePtrs,
		const SmallVectorImpl< Value *> &	LiveVariables,
		PartiallyConstructedSafepointRecord &	Result,
		std::vector< DeferredReplacement > &	Replacements
	)

static

Definition at line 1419 of file RewriteStatepointsForGC.cpp.

Referenced by makeStatepointExplicit().

◆ meetBDVState()

static BDVState meetBDVState	(	const BDVState &	LHS,
		const BDVState &	RHS
	)

static

Definition at line 755 of file RewriteStatepointsForGC.cpp.

References assert(), and meetBDVStateImpl().

Referenced by findBasePointer().

◆ meetBDVStateImpl()

static BDVState meetBDVStateImpl	(	const BDVState &	LHS,
		const BDVState &	RHS
	)

static

Definition at line 727 of file RewriteStatepointsForGC.cpp.

References assert(), llvm_unreachable, and llvm::Unknown.

Referenced by meetBDVState().

◆ normalizeForInvokeSafepoint()

static BasicBlock* normalizeForInvokeSafepoint	(	BasicBlock *	BB,
		BasicBlock *	InvokeParent,
		DominatorTree &	DT
	)

static

Definition at line 1226 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::BasicBlock::begin(), llvm::FoldSingleEntryPHINodes(), llvm::BasicBlock::getUniquePredecessor(), llvm::MipsISD::Ret, and llvm::SplitBlockPredecessors().

Referenced by insertParsePoints().

◆ operator<<()

static raw_ostream& operator<<	(	raw_ostream &	OS,
		const BDVState &	State
	)

static

Definition at line 721 of file RewriteStatepointsForGC.cpp.

◆ recomputeLiveInValues() [1/2]

static void recomputeLiveInValues	(	GCPtrLivenessData &	RevisedLivenessData,
		CallSite	CS,
		PartiallyConstructedSafepointRecord &	result
	)

static

Given an updated version of the dataflow liveness results, update the liveset and base pointer maps for the call site CS.

Definition at line 2788 of file RewriteStatepointsForGC.cpp.

References assert(), findLiveSetAtInst(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getInstruction(), llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::insert(), and isKnownBaseResult().

Referenced by findBasePointers(), insertParsePoints(), and recomputeLiveInValues().

◆ recomputeLiveInValues() [2/2]

static void recomputeLiveInValues	(	Function &	F,
		DominatorTree &	DT,
		ArrayRef< CallSite >	toUpdate,
		MutableArrayRef< struct PartiallyConstructedSafepointRecord >	records
	)

static

Definition at line 1206 of file RewriteStatepointsForGC.cpp.

References computeLiveInValues(), info, recomputeLiveInValues(), and llvm::ArrayRef< T >::size().

◆ relocationViaAlloca()

static void relocationViaAlloca	(	Function &	F,
		DominatorTree &	DT,
		ArrayRef< Value *>	Live,
		ArrayRef< PartiallyConstructedSafepointRecord >	Records
	)

static

Do all the relocation update via allocas and mem2reg.

Definition at line 1694 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T >::begin(), llvm::detail::DenseSetImpl< ValueT, MapTy, ValueInfoT >::count(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::tgtok::Def, llvm::SmallVectorBase::empty(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::ConstantPointerNull::get(), llvm::DataLayout::getAllocaAddrSpace(), llvm::Module::getDataLayout(), llvm::Function::getEntryBlock(), llvm::BasicBlock::getFirstNonPHI(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::PHINode::getNumIncomingValues(), llvm::Value::getNumUses(), llvm::GlobalValue::getParent(), llvm::BasicBlock::getTerminator(), Info, llvm::Instruction::insertAfter(), llvm::Instruction::insertBefore(), insertRelocationStores(), insertRematerializationStores(), llvm::SPII::Load, llvm::PromoteMemToReg(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SmallVectorImpl< T >::reserve(), llvm::PHINode::setIncomingValue(), llvm::SmallVectorBase::size(), llvm::ArrayRef< T >::size(), llvm::sort(), llvm::SPII::Store, and llvm::Value::users().

Referenced by insertParsePoints().

◆ rematerializeLiveValues()

static void rematerializeLiveValues	(	CallSite	CS,
		PartiallyConstructedSafepointRecord &	Info,
		TargetTransformInfo &	TTI
	)

static

Definition at line 2025 of file RewriteStatepointsForGC.cpp.

References AreEquivalentPhiNodes(), assert(), llvm::SmallVectorTemplateCommon< T >::begin(), chainToBasePointerCost(), llvm::Instruction::clone(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T >::end(), findRematerializableChainToBasePointer(), llvm::BasicBlock::getFirstInsertionPt(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::getInstruction(), llvm::ilist_node_with_parent< NodeTy, ParentTy, Options >::getNextNode(), llvm::InvokeInst::getNormalDest(), llvm::InvokeInst::getUnwindDest(), llvm::Instruction::insertBefore(), llvm::is_contained(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::isCall(), llvm::CallSiteBase< FunTy, BBTy, ValTy, UserTy, UseTy, InstrTy, CallTy, InvokeTy, IterTy >::isInvoke(), llvm::User::operand_values(), llvm::SmallVectorTemplateBase< T >::push_back(), RematerializationThreshold, llvm::User::replaceUsesOfWith(), llvm::reverse(), llvm::Value::setName(), and llvm::SmallVectorBase::size().

Referenced by insertParsePoints().

◆ RemoveNonValidAttrAtIndex()

template<typename AttrHolder >

static void RemoveNonValidAttrAtIndex	(	LLVMContext &	Ctx,
		AttrHolder &	AH,
		unsigned	Index
	)

static

Definition at line 2389 of file RewriteStatepointsForGC.cpp.

References llvm::AttrBuilder::addAttribute(), llvm::Attribute::Dereferenceable, llvm::Attribute::DereferenceableOrNull, llvm::AttrBuilder::empty(), llvm::Attribute::get(), llvm::Attribute::NoAlias, and llvm::RISCVFenceField::R.

Referenced by stripNonValidAttributesFromPrototype(), and stripNonValidDataFromBody().

◆ shouldRewriteStatepointsIn()

static bool shouldRewriteStatepointsIn ( Function & F )

static

Returns true if this function should be rewritten by this pass.

The main point of this function is as an extension point for custom logic.

Definition at line 2497 of file RewriteStatepointsForGC.cpp.

References llvm::Function::getGC(), and llvm::Function::hasGC().

Referenced by llvm::RewriteStatepointsForGC::run(), llvm::RewriteStatepointsForGC::runOnFunction(), and stripNonValidData().

◆ stripInvalidMetadataFromInstruction()

static void stripInvalidMetadataFromInstruction ( Instruction & I )

static

Certain metadata on instructions are invalid after running RS4GC.

Optimizations that run after RS4GC can incorrectly use this metadata to optimize functions. We drop such metadata on the instruction.

Definition at line 2420 of file RewriteStatepointsForGC.cpp.

References llvm::Instruction::dropUnknownNonDebugMetadata(), llvm::LLVMContext::MD_alias_scope, llvm::LLVMContext::MD_align, llvm::LLVMContext::MD_nonnull, llvm::LLVMContext::MD_nontemporal, llvm::LLVMContext::MD_range, llvm::LLVMContext::MD_tbaa, and llvm::LLVMContext::MD_type.

Referenced by stripNonValidDataFromBody().

◆ stripNonValidAttributesFromPrototype()

static void stripNonValidAttributesFromPrototype ( Function & F )

static

Definition at line 2405 of file RewriteStatepointsForGC.cpp.

References A, llvm::Function::args(), llvm::AttributeList::FirstArgIndex, llvm::Function::getContext(), llvm::Function::getReturnType(), RemoveNonValidAttrAtIndex(), and llvm::AttributeList::ReturnIndex.

Referenced by stripNonValidData().

◆ stripNonValidData()

static void stripNonValidData ( Module & M )

static

The IR fed into RewriteStatepointsForGC may have had attributes and metadata implying dereferenceability that are no longer valid/correct after RewriteStatepointsForGC has run.

This is because semantically, after RewriteStatepointsForGC runs, all calls to gc.statepoint "free" the entire heap. stripNonValidData (conservatively) restores correctness by erasing all attributes in the module that externally imply dereferenceability. Similar reasoning also applies to the noalias attributes and metadata. gc.statepoint can touch the entire heap including noalias objects. Apart from attributes and metadata, we also remove instructions that imply constant physical memory: llvm.invariant.start.

Definition at line 2509 of file RewriteStatepointsForGC.cpp.

References llvm::any_of(), assert(), shouldRewriteStatepointsIn(), stripNonValidAttributesFromPrototype(), and stripNonValidDataFromBody().

Referenced by llvm::RewriteStatepointsForGC::run().