AtomicExpandPass.cpp File Reference

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/AtomicExpandUtils.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include <cassert>
#include <cstdint>
#include <iterator>

Include dependency graph for AtomicExpandPass.cpp:

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "atomic-expand"

Functions
	INITIALIZE_PASS (AtomicExpand, DEBUG_TYPE, "Expand Atomic instructions", false, false) FunctionPass *llvm
static unsigned	getAtomicOpSize (LoadInst *LI)
static unsigned	getAtomicOpSize (StoreInst *SI)
static unsigned	getAtomicOpSize (AtomicRMWInst *RMWI)
static unsigned	getAtomicOpSize (AtomicCmpXchgInst *CASI)
static unsigned	getAtomicOpAlign (LoadInst *LI)
static unsigned	getAtomicOpAlign (StoreInst *SI)
static unsigned	getAtomicOpAlign (AtomicRMWInst *RMWI)
static unsigned	getAtomicOpAlign (AtomicCmpXchgInst *CASI)
template<typename Inst >
static bool	atomicSizeSupported (const TargetLowering *TLI, Inst *I)
static void	createCmpXchgInstFun (IRBuilder<> &Builder, Value *Addr, Value *Loaded, Value *NewVal, AtomicOrdering MemOpOrder, Value *&Success, Value *&NewLoaded)
static Value *	performAtomicOp (AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, Value *Loaded, Value *Inc)
	Emit IR to implement the given atomicrmw operation on values in registers, returning the new value. More...
static PartwordMaskValues	createMaskInstrs (IRBuilder<> &Builder, Instruction *I, Type *ValueType, Value *Addr, unsigned WordSize)
	This is a helper function which builds instructions to provide values necessary for partword atomic operations. More...
static Value *	performMaskedAtomicOp (AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, Value *Loaded, Value *Shifted_Inc, Value *Inc, const PartwordMaskValues &PMV)
	Emit IR to implement a masked version of a given atomicrmw operation. More...
static bool	canUseSizedAtomicCall (unsigned Size, unsigned Align, const DataLayout &DL)
static ArrayRef< RTLIB::Libcall >	GetRMWLibcall (AtomicRMWInst::BinOp Op)

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "atomic-expand"

Definition at line 55 of file AtomicExpandPass.cpp.

Function Documentation

atomicSizeSupported()

template<typename Inst >

static bool atomicSizeSupported ( const TargetLowering *  TLI, Inst *  I  )

static

Definition at line 192 of file AtomicExpandPass.cpp.

References llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, llvm::AtomicRMWInst::And, assert(), llvm::TargetLoweringBase::CmpXChg, llvm::IRBuilder< T, Inserter >::CreateAtomicCmpXchg(), llvm::IRBuilder< T, Inserter >::CreateAtomicRMW(), llvm::IRBuilder< T, Inserter >::CreateBitCast(), llvm::IRBuilder< T, Inserter >::CreateExtractValue(), llvm::IRBuilder< T, Inserter >::CreateStore(), llvm::dbgs(), llvm::dyn_cast(), E, llvm::Instruction::eraseFromParent(), F(), llvm::IntegerType::get(), llvm::PointerType::get(), llvm::LoadInst::getAlignment(), llvm::StoreInst::getAlignment(), getAtomicOpAlign(), getAtomicOpSize(), llvm::Type::getContext(), llvm::TargetLoweringBase::getMaxAtomicSizeInBitsSupported(), llvm::Instruction::getModule(), llvm::Constant::getNullValue(), llvm::LoadInst::getOrdering(), llvm::StoreInst::getOrdering(), llvm::Type::getPointerAddressSpace(), llvm::LoadInst::getPointerOperand(), llvm::StoreInst::getPointerOperand(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(), llvm::LoadInst::getSyncScopeID(), llvm::StoreInst::getSyncScopeID(), llvm::Value::getType(), llvm::StoreInst::getValueOperand(), I, llvm::inst_begin(), llvm::inst_end(), llvm::isAcquireOrStronger(), llvm::Instruction::isAtomic(), llvm::isReleaseOrStronger(), llvm::LoadInst::isVolatile(), llvm::StoreInst::isVolatile(), llvm::TargetLoweringBase::LLOnly, llvm::TargetLoweringBase::LLSC, LLVM_DEBUG, llvm_unreachable, llvm::Monotonic, llvm::TargetLoweringBase::None, llvm::AtomicRMWInst::Or, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::Value::replaceAllUsesWith(), runOnFunction(), llvm::StoreInst::setAlignment(), llvm::StoreInst::setAtomic(), llvm::StoreInst::setVolatile(), SI, Size, llvm::SystemZISD::TM, llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

canUseSizedAtomicCall()

static bool canUseSizedAtomicCall ( unsigned  Size, unsigned  Align, const DataLayout &  DL  )

static

Definition at line 1426 of file AtomicExpandPass.cpp.

References llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, assert(), llvm::ISD::ATOMIC_LOAD, llvm::ISD::ATOMIC_STORE, getAtomicOpAlign(), getAtomicOpSize(), llvm::AtomicCmpXchgInst::getCompareOperand(), llvm::AtomicCmpXchgInst::getFailureOrdering(), llvm::DataLayout::getLargestLegalIntTypeSizeInBits(), llvm::AtomicCmpXchgInst::getNewValOperand(), llvm::LoadInst::getOrdering(), llvm::StoreInst::getOrdering(), llvm::LoadInst::getPointerOperand(), llvm::StoreInst::getPointerOperand(), llvm::AtomicCmpXchgInst::getPointerOperand(), llvm::AtomicCmpXchgInst::getSuccessOrdering(), llvm::StoreInst::getValueOperand(), llvm::NotAtomic, and Size.

Referenced by GetRMWLibcall().

createCmpXchgInstFun()

static void createCmpXchgInstFun ( IRBuilder<> &  Builder, Value *  Addr, Value *  Loaded, Value *  NewVal, AtomicOrdering  MemOpOrder, Value *&  Success, Value *&  NewLoaded  )

static

Definition at line 495 of file AtomicExpandPass.cpp.

References llvm::IRBuilder< T, Inserter >::CreateAtomicCmpXchg(), llvm::IRBuilder< T, Inserter >::CreateExtractValue(), and llvm::AtomicCmpXchgInst::getStrongestFailureOrdering().

Referenced by performAtomicOp(), and performMaskedAtomicOp().

createMaskInstrs()

static PartwordMaskValues createMaskInstrs ( IRBuilder<> &  Builder, Instruction *  I, Type *  ValueType, Value *  Addr, unsigned  WordSize  )

static

This is a helper function which builds instructions to provide values necessary for partword atomic operations.

It takes an incoming address, Addr, and ValueType, and constructs the address, shift-amounts and masks needed to work with a larger value of size WordSize.

AlignedAddr: Addr rounded down to a multiple of WordSize

ShiftAmt: Number of bits to right-shift a WordSize value loaded from AlignAddr for it to have the same value as if ValueType was loaded from Addr.

Mask: Value to mask with the value loaded from AlignAddr to include only the part that would've been loaded from Addr.

Inv_Mask: The inverse of Mask.

Definition at line 613 of file AtomicExpandPass.cpp.

References assert(), llvm::IRBuilder< T, Inserter >::CreateAnd(), llvm::IRBuilder< T, Inserter >::CreateIntToPtr(), llvm::IRBuilder< T, Inserter >::CreateNot(), llvm::IRBuilder< T, Inserter >::CreatePtrToInt(), llvm::IRBuilder< T, Inserter >::CreateShl(), llvm::IRBuilder< T, Inserter >::CreateTrunc(), llvm::IRBuilder< T, Inserter >::CreateXor(), llvm::ConstantInt::get(), llvm::Function::getContext(), llvm::Module::getDataLayout(), llvm::Type::getIntNTy(), llvm::DataLayout::getIntPtrType(), llvm::Instruction::getModule(), llvm::Instruction::getParent(), llvm::BasicBlock::getParent(), llvm::Type::getPointerAddressSpace(), llvm::Value::getType(), llvm::DataLayout::getTypeStoreSize(), llvm::DataLayout::isLittleEndian(), and llvm::MipsISD::Ret.

Referenced by performMaskedAtomicOp().

getAtomicOpAlign() [1/4]

static unsigned getAtomicOpAlign ( LoadInst *  LI )

static

Definition at line 156 of file AtomicExpandPass.cpp.

References llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, assert(), and llvm::LoadInst::getAlignment().

Referenced by atomicSizeSupported(), canUseSizedAtomicCall(), and GetRMWLibcall().

getAtomicOpAlign() [2/4]

static unsigned getAtomicOpAlign ( StoreInst *  SI )

static

Definition at line 165 of file AtomicExpandPass.cpp.

References llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, assert(), and llvm::StoreInst::getAlignment().

getAtomicOpAlign() [3/4]

static unsigned getAtomicOpAlign ( AtomicRMWInst *  RMWI )

static

Definition at line 174 of file AtomicExpandPass.cpp.

References llvm::Module::getDataLayout(), llvm::Instruction::getModule(), llvm::Value::getType(), llvm::DataLayout::getTypeStoreSize(), and llvm::AtomicRMWInst::getValOperand().

getAtomicOpAlign() [4/4]

static unsigned getAtomicOpAlign ( AtomicCmpXchgInst *  CASI )

static

Definition at line 182 of file AtomicExpandPass.cpp.

References llvm::AtomicCmpXchgInst::getCompareOperand(), llvm::Module::getDataLayout(), llvm::Instruction::getModule(), llvm::Value::getType(), and llvm::DataLayout::getTypeStoreSize().

getAtomicOpSize() [1/4]

static unsigned getAtomicOpSize ( LoadInst *  LI )

static

Definition at line 135 of file AtomicExpandPass.cpp.

References llvm::Module::getDataLayout(), llvm::Instruction::getModule(), llvm::Value::getType(), and llvm::DataLayout::getTypeStoreSize().

Referenced by atomicSizeSupported(), canUseSizedAtomicCall(), GetRMWLibcall(), performAtomicOp(), and performMaskedAtomicOp().

getAtomicOpSize() [2/4]

static unsigned getAtomicOpSize ( StoreInst *  SI )

static

Definition at line 140 of file AtomicExpandPass.cpp.

References llvm::Module::getDataLayout(), llvm::Instruction::getModule(), llvm::Value::getType(), llvm::DataLayout::getTypeStoreSize(), and llvm::StoreInst::getValueOperand().

getAtomicOpSize() [3/4]

static unsigned getAtomicOpSize ( AtomicRMWInst *  RMWI )

static

Definition at line 145 of file AtomicExpandPass.cpp.

References llvm::Module::getDataLayout(), llvm::Instruction::getModule(), llvm::Value::getType(), llvm::DataLayout::getTypeStoreSize(), and llvm::AtomicRMWInst::getValOperand().

getAtomicOpSize() [4/4]

static unsigned getAtomicOpSize ( AtomicCmpXchgInst *  CASI )

static

Definition at line 150 of file AtomicExpandPass.cpp.

References llvm::AtomicCmpXchgInst::getCompareOperand(), llvm::Module::getDataLayout(), llvm::Instruction::getModule(), llvm::Value::getType(), and llvm::DataLayout::getTypeStoreSize().

GetRMWLibcall()

static ArrayRef<RTLIB::Libcall> GetRMWLibcall ( AtomicRMWInst::BinOp  Op )

static

Definition at line 1485 of file AtomicExpandPass.cpp.

References llvm::AtomicRMWInst::Add, llvm::AttributeList::addAttribute(), llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align, llvm::AtomicRMWInst::And, Arg, llvm::AMDGPU::HSAMD::Kernel::Key::Args, assert(), llvm::AtomicRMWInst::BAD_BINOP, canUseSizedAtomicCall(), llvm::IRBuilder< T, Inserter >::CreateAlignedLoad(), llvm::IRBuilder< T, Inserter >::CreateAlignedStore(), llvm::IRBuilder< T, Inserter >::CreateAtomicCmpXchg(), llvm::IRBuilder< T, Inserter >::CreateBitCast(), llvm::IRBuilder< T, Inserter >::CreateBitOrPointerCast(), llvm::IRBuilder< T, Inserter >::CreateCall(), llvm::IRBuilder< T, Inserter >::CreateExtractValue(), llvm::IRBuilder< T, Inserter >::CreateInsertValue(), llvm::IRBuilderBase::CreateLifetimeEnd(), llvm::IRBuilderBase::CreateLifetimeStart(), llvm::ArrayRef< T >::empty(), llvm::Instruction::eraseFromParent(), llvm::expandAtomicRMWToCmpXchg(), llvm::BasicBlock::front(), llvm::ConstantInt::get(), llvm::FunctionType::get(), llvm::UndefValue::get(), getAtomicOpAlign(), getAtomicOpSize(), llvm::Value::getContext(), llvm::Module::getDataLayout(), llvm::Function::getEntryBlock(), llvm::Instruction::getFunction(), llvm::Type::getInt1Ty(), llvm::Type::getInt32Ty(), llvm::Type::getInt64Ty(), llvm::Type::getInt8PtrTy(), llvm::Type::getIntNTy(), llvm::DataLayout::getIntPtrType(), llvm::Instruction::getModule(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOrdering(), llvm::Module::getOrInsertFunction(), llvm::AtomicRMWInst::getPointerOperand(), llvm::DataLayout::getPrefTypeAlignment(), llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(), llvm::Value::getType(), llvm::AtomicRMWInst::getValOperand(), llvm::Type::getVoidTy(), llvm_unreachable, llvm::makeArrayRef(), llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::NotAtomic, llvm::AtomicRMWInst::Or, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::Value::replaceAllUsesWith(), llvm::AttributeList::ReturnIndex, llvm::AllocaInst::setAlignment(), llvm::CallBase::setAttributes(), Size, llvm::ArrayRef< T >::size(), llvm::AtomicRMWInst::Sub, Success, llvm::toCABI(), llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::AtomicRMWInst::Xchg, llvm::AtomicRMWInst::Xor, and llvm::Attribute::ZExt.

INITIALIZE_PASS()

INITIALIZE_PASS	(	AtomicExpand	,
		DEBUG_TYPE	,
		"Expand Atomic instructions"	,
		false	,
		false
	)

Definition at line 129 of file AtomicExpandPass.cpp.

performAtomicOp()

static Value* performAtomicOp ( AtomicRMWInst::BinOp  Op, IRBuilder<> &  Builder, Value *  Loaded, Value *  Inc  )

static

Emit IR to implement the given atomicrmw operation on values in registers, returning the new value.

Definition at line 508 of file AtomicExpandPass.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::TargetLoweringBase::CmpXChg, llvm::IRBuilder< T, Inserter >::CreateAdd(), llvm::IRBuilder< T, Inserter >::CreateAnd(), createCmpXchgInstFun(), llvm::IRBuilder< T, Inserter >::CreateICmpSGT(), llvm::IRBuilder< T, Inserter >::CreateICmpSLE(), llvm::IRBuilder< T, Inserter >::CreateICmpUGT(), llvm::IRBuilder< T, Inserter >::CreateICmpULE(), llvm::IRBuilder< T, Inserter >::CreateNot(), llvm::IRBuilder< T, Inserter >::CreateOr(), llvm::IRBuilder< T, Inserter >::CreateSelect(), llvm::IRBuilder< T, Inserter >::CreateSub(), llvm::IRBuilder< T, Inserter >::CreateXor(), llvm::expandAtomicRMWToCmpXchg(), getAtomicOpSize(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOrdering(), llvm::AtomicRMWInst::getPointerOperand(), llvm::Value::getType(), llvm::AtomicRMWInst::getValOperand(), llvm::TargetLoweringBase::LLSC, llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::TargetLoweringBase::MaskedIntrinsic, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::TargetLoweringBase::None, llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.

Referenced by llvm::expandAtomicRMWToCmpXchg(), and performMaskedAtomicOp().

performMaskedAtomicOp()

static Value* performMaskedAtomicOp ( AtomicRMWInst::BinOp  Op, IRBuilder<> &  Builder, Value *  Loaded, Value *  Shifted_Inc, Value *  Inc, const PartwordMaskValues &  PMV  )

static

Emit IR to implement a masked version of a given atomicrmw operation.

(That is, only the bits under the Mask should be affected by the operation)

Definition at line 662 of file AtomicExpandPass.cpp.

References llvm::Acquire, llvm::AtomicRMWInst::Add, llvm::PHINode::addIncoming(), llvm::AtomicRMWInst::And, assert(), llvm::BasicBlock::begin(), C, llvm::TargetLoweringBase::CmpXChg, llvm::BasicBlock::Create(), llvm::IRBuilder< T, Inserter >::CreateAnd(), llvm::IRBuilder< T, Inserter >::CreateAtomicCmpXchg(), llvm::IRBuilder< T, Inserter >::CreateAtomicRMW(), llvm::IRBuilder< T, Inserter >::CreateBr(), llvm::IRBuilder< T, Inserter >::CreateCast(), createCmpXchgInstFun(), llvm::IRBuilder< T, Inserter >::CreateCondBr(), llvm::IRBuilder< T, Inserter >::CreateExtractValue(), llvm::IRBuilder< T, Inserter >::CreateICmpEQ(), llvm::IRBuilder< T, Inserter >::CreateICmpNE(), llvm::IRBuilder< T, Inserter >::CreateInsertValue(), llvm::IRBuilder< T, Inserter >::CreateLoad(), llvm::IRBuilder< T, Inserter >::CreateLShr(), createMaskInstrs(), llvm::IRBuilder< T, Inserter >::CreateOr(), llvm::IRBuilder< T, Inserter >::CreatePHI(), llvm::IRBuilder< T, Inserter >::CreateShl(), llvm::IRBuilder< T, Inserter >::CreateTrunc(), llvm::IRBuilder< T, Inserter >::CreateUnreachable(), llvm::IRBuilder< T, Inserter >::CreateZExt(), llvm::dbgs(), llvm::dyn_cast(), llvm::BasicBlock::end(), llvm::Instruction::eraseFromParent(), llvm::IntegerType::get(), llvm::ConstantInt::get(), llvm::PointerType::get(), llvm::UndefValue::get(), getAtomicOpSize(), llvm::AtomicCmpXchgInst::getCompareOperand(), llvm::IRBuilderBase::getContext(), llvm::Function::getContext(), llvm::AtomicCmpXchgInst::getFailureOrdering(), llvm::ConstantInt::getFalse(), llvm::ExtractValueInst::getIndices(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::GetInsertPoint(), llvm::Type::getInt1Ty(), llvm::Type::getInt32Ty(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::Instruction::getModule(), llvm::AtomicCmpXchgInst::getNewValOperand(), llvm::ExtractValueInst::getNumIndices(), llvm::AtomicRMWInst::getOperation(), llvm::AtomicRMWInst::getOrdering(), llvm::Instruction::getParent(), llvm::BasicBlock::getParent(), llvm::Type::getPointerAddressSpace(), llvm::AtomicCmpXchgInst::getPointerOperand(), llvm::AtomicRMWInst::getPointerOperand(), llvm::Type::getPrimitiveSizeInBits(), llvm::AtomicCmpXchgInst::getSuccessOrdering(), llvm::AtomicCmpXchgInst::getSyncScopeID(), llvm::ConstantInt::getTrue(), llvm::Value::getType(), llvm::AtomicRMWInst::getValOperand(), llvm::AtomicCmpXchgInst::isVolatile(), llvm::AtomicCmpXchgInst::isWeak(), llvm::TargetLoweringBase::LLSC, LLVM_DEBUG, llvm_unreachable, llvm::TargetLoweringBase::MaskedIntrinsic, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::Monotonic, llvm::AtomicRMWInst::Nand, llvm::TargetLoweringBase::None, llvm::Function::optForMinSize(), llvm::AtomicRMWInst::Or, performAtomicOp(), llvm::SmallVectorTemplateBase< T >::push_back(), llvm::Value::replaceAllUsesWith(), llvm::LoadInst::setAlignment(), llvm::IRBuilderBase::SetInsertPoint(), llvm::LoadInst::setVolatile(), llvm::AtomicCmpXchgInst::setVolatile(), llvm::AtomicCmpXchgInst::setWeak(), llvm::BasicBlock::splitBasicBlock(), llvm::AtomicRMWInst::Sub, Success, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, llvm::Unordered, llvm::Value::use_empty(), llvm::Value::users(), llvm::AtomicRMWInst::Xchg, and llvm::AtomicRMWInst::Xor.