31#include "llvm/IR/IntrinsicsSPIRV.h"
40class SPIRVPrepareFunctions :
public ModulePass {
41 const SPIRVTargetMachine &TM;
42 bool substituteIntrinsicCalls(Function *
F);
43 Function *removeAggregateTypesFromSignature(Function *
F);
47 SPIRVPrepareFunctions(
const SPIRVTargetMachine &TM)
48 : ModulePass(ID), TM(TM) {}
50 bool runOnModule(
Module &M)
override;
52 StringRef getPassName()
const override {
return "SPIRV prepare functions"; }
54 void getAnalysisUsage(AnalysisUsage &AU)
const override {
55 ModulePass::getAnalysisUsage(AU);
61 cl::desc(
"Emit unknown intrinsics as calls to external functions. A "
62 "comma-separated input list of intrinsic prefixes must be "
63 "provided, and only intrinsics carrying a listed prefix get "
64 "emitted as described."),
68char SPIRVPrepareFunctions::ID = 0;
71 "SPIRV prepare functions",
false,
false)
74 Function *IntrinsicFunc =
II->getCalledFunction();
75 assert(IntrinsicFunc &&
"Missing function");
76 std::string FuncName = IntrinsicFunc->
getName().
str();
78 FuncName =
"spirv." + FuncName;
87 if (
F &&
F->getFunctionType() == FT)
106 std::string FuncName = lowerLLVMIntrinsicName(
Intrinsic);
108 FuncName +=
".volatile";
117 M->getOrInsertFunction(FuncName,
Intrinsic->getFunctionType());
118 auto IntrinsicID =
Intrinsic->getIntrinsicID();
122 assert(
F &&
"Callee must be a function");
124 switch (IntrinsicID) {
125 case Intrinsic::memset: {
134 IsVolatile->setName(
"isvolatile");
137 auto *MemSet = IRB.
CreateMemSet(Dest, Val, Len, MSI->getDestAlign(),
141 MemSet->eraseFromParent();
144 case Intrinsic::bswap: {
162 AnnoVal =
Ref->getOperand(0);
164 OptAnnoVal =
Ref->getOperand(0);
175 C &&
C->getNumOperands()) {
176 Value *MaybeStruct =
C->getOperand(0);
178 for (
unsigned I = 0,
E =
Struct->getNumOperands();
I !=
E; ++
I) {
180 Anno += (
I == 0 ?
": " :
", ") +
181 std::to_string(CInt->getType()->getIntegerBitWidth() == 1
182 ? CInt->getZExtValue()
183 : CInt->getSExtValue());
187 for (
unsigned I = 0,
E =
Struct->getType()->getStructNumElements();
189 Anno +=
I == 0 ?
": 0" :
", 0";
196 const std::string &Anno,
203 static const std::regex R(
204 "\\{(\\d+)(?:[:,](\\d+|\"[^\"]*\")(?:,(\\d+|\"[^\"]*\"))*)?\\}");
207 for (std::sregex_iterator
208 It = std::sregex_iterator(Anno.begin(), Anno.end(), R),
209 ItEnd = std::sregex_iterator();
211 if (It->position() != Pos)
213 Pos = It->position() + It->length();
214 std::smatch Match = *It;
216 for (std::size_t i = 1; i < Match.size(); ++i) {
217 std::ssub_match SMatch = Match[i];
218 std::string Item = SMatch.str();
219 if (Item.length() == 0)
221 if (Item[0] ==
'"') {
222 Item = Item.substr(1, Item.length() - 2);
224 static const std::regex RStr(
"^(\\d+)(?:,(\\d+))*$");
225 if (std::smatch MatchStr; std::regex_match(Item, MatchStr, RStr)) {
226 for (std::size_t SubIdx = 1; SubIdx < MatchStr.size(); ++SubIdx)
227 if (std::string SubStr = MatchStr[SubIdx].str(); SubStr.length())
229 ConstantInt::get(
Int32Ty, std::stoi(SubStr))));
240 if (MDsItem.
size() == 0)
244 return Pos ==
static_cast<int>(Anno.length()) ? std::move(MDs)
253 Value *PtrArg =
nullptr;
255 PtrArg = BI->getOperand(0);
257 PtrArg =
II->getOperand(0);
260 4 <
II->arg_size() ?
II->getArgOperand(4) :
nullptr);
269 if (MDs.
size() == 0) {
279 Intrinsic::spv_assign_decoration, {PtrArg->
getType()},
281 II->replaceAllUsesWith(
II->getOperand(0));
291 Type *FSHRetTy = FSHFuncTy->getReturnType();
292 const std::string FuncName = lowerLLVMIntrinsicName(FSHIntrinsic);
296 if (!FSHFunc->
empty()) {
308 unsigned BitWidth = IntTy->getIntegerBitWidth();
310 Value *BitWidthForInsts =
314 Value *RotateModVal =
316 Value *FirstShift =
nullptr, *SecShift =
nullptr;
329 Value *SubRotateVal = IRB.
CreateSub(BitWidthForInsts, RotateModVal);
348 if (!ConstrainedCmpIntrinsic)
370 if (
II->getIntrinsicID() == Intrinsic::assume) {
372 II->getModule(), Intrinsic::SPVIntrinsics::spv_assume);
373 II->setCalledFunction(
F);
374 }
else if (
II->getIntrinsicID() == Intrinsic::expect) {
376 II->getModule(), Intrinsic::SPVIntrinsics::spv_expect,
377 {II->getOperand(0)->getType()});
378 II->setCalledFunction(
F);
387 std::optional<TypeSize>
Size =
388 Alloca->getAllocationSize(Alloca->getDataLayout());
390 Builder.CreateIntrinsic(NewID, Alloca->getType(),
391 {SizeVal, II->getArgOperand(0)});
392 II->eraseFromParent();
398bool SPIRVPrepareFunctions::substituteIntrinsicCalls(
Function *
F) {
400 const SPIRVSubtarget &STI = TM.
getSubtarget<SPIRVSubtarget>(*F);
402 for (BasicBlock &BB : *
F) {
411 switch (
II->getIntrinsicID()) {
412 case Intrinsic::memset:
413 case Intrinsic::bswap:
416 case Intrinsic::fshl:
417 case Intrinsic::fshr:
421 case Intrinsic::assume:
422 case Intrinsic::expect:
427 case Intrinsic::lifetime_start:
430 II, Intrinsic::SPVIntrinsics::spv_lifetime_start);
432 II->eraseFromParent();
436 case Intrinsic::lifetime_end:
439 II, Intrinsic::SPVIntrinsics::spv_lifetime_end);
441 II->eraseFromParent();
445 case Intrinsic::ptr_annotation:
449 case Intrinsic::experimental_constrained_fcmp:
450 case Intrinsic::experimental_constrained_fcmps:
457 any_of(SPVAllowUnknownIntrinsics, [
II](
auto &&Prefix) {
460 return II->getCalledFunction()->getName().starts_with(Prefix);
467 for (
auto *
I : EraseFromParent)
468 I->eraseFromParent();
476SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *
F) {
477 bool IsRetAggr =
F->getReturnType()->isAggregateType();
479 if (
F->isIntrinsic() && IsRetAggr)
484 bool HasAggrArg =
llvm::any_of(
F->args(), [](Argument &Arg) {
485 return Arg.getType()->isAggregateType();
487 bool DoClone = IsRetAggr || HasAggrArg;
491 Type *RetType = IsRetAggr ?
B.getInt32Ty() :
F->getReturnType();
493 ChangedTypes.
push_back(std::pair<int, Type *>(-1,
F->getReturnType()));
495 for (
const auto &Arg :
F->args()) {
496 if (Arg.getType()->isAggregateType()) {
499 std::pair<int, Type *>(Arg.getArgNo(), Arg.getType()));
503 FunctionType *NewFTy =
504 FunctionType::get(RetType, ArgTypes,
F->getFunctionType()->isVarArg());
510 for (
auto &Arg :
F->args()) {
511 StringRef ArgName = Arg.getName();
512 NewFArgIt->setName(ArgName);
513 VMap[&Arg] = &(*NewFArgIt++);
521 NamedMDNode *FuncMD =
522 F->getParent()->getOrInsertNamedMetadata(
"spv.cloned_funcs");
525 for (
auto &ChangedTyP : ChangedTypes)
528 {ConstantAsMetadata::get(B.getInt32(ChangedTyP.first)),
529 ValueAsMetadata::get(Constant::getNullValue(ChangedTyP.second))}));
530 MDNode *ThisFuncMD =
MDNode::get(
B.getContext(), MDArgs);
536 U->replaceUsesOfWith(
F, NewF);
540 if (RetType !=
F->getReturnType())
541 TM.
getSubtarget<SPIRVSubtarget>(*F).getSPIRVGlobalRegistry()->addMutated(
542 NewF,
F->getReturnType());
546bool SPIRVPrepareFunctions::runOnModule(
Module &M) {
548 for (Function &
F : M) {
549 Changed |= substituteIntrinsicCalls(&
F);
553 std::vector<Function *> FuncsWorklist;
555 FuncsWorklist.push_back(&
F);
557 for (
auto *
F : FuncsWorklist) {
558 Function *NewF = removeAggregateTypesFromSignature(
F);
561 F->eraseFromParent();
570 return new SPIRVPrepareFunctions(TM);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Machine Check Debug Module
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic)
static std::string getAnnotation(Value *AnnoVal, Value *OptAnnoVal)
static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic)
static void lowerConstrainedFPCmpIntrinsic(ConstrainedFPCmpIntrinsic *ConstrainedCmpIntrinsic, SmallVector< Instruction * > &EraseFromParent)
static void lowerPtrAnnotation(IntrinsicInst *II)
static SmallVector< Metadata * > parseAnnotation(Value *I, const std::string &Anno, LLVMContext &Ctx, Type *Int32Ty)
static bool toSpvLifetimeIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID)
static void lowerExpectAssume(IntrinsicInst *II)
static Function * getOrCreateFunction(Module *M, Type *RetTy, ArrayRef< Type * > ArgTypes, StringRef Name)
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
This is the shared class of boolean and integer constants.
Constrained floating point compare intrinsics.
LLVM_ABI FCmpInst::Predicate getPredicate() const
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
Type * getReturnType() const
Returns the type of the ret val.
void setCallingConv(CallingConv::ID CC)
Argument * getArg(unsigned i) const
void setDSOLocal(bool Local)
@ ExternalLinkage
Externally visible function.
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
ReturnInst * CreateRet(Value *V)
Create a 'ret <val>' instruction.
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, const AAMDNodes &AAInfo=AAMDNodes())
Create and insert a memset to the specified pointer and the specified value.
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
void LowerIntrinsicCall(CallInst *CI)
Replace a call to the specified intrinsic function.
This is an important class for using LLVM in a threaded context.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
A Module instance is used to store all the information related to an LLVM module.
LLVM_ABI void addOperand(MDNode *M)
bool canUseExtension(SPIRV::Extension::Extension E) const
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
const Triple & getTargetTriple() const
const STC & getSubtarget(const Function &F) const
This method returns a pointer to the specified type of TargetSubtargetInfo.
VendorType getVendor() const
Get the parsed vendor type of this triple.
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Type * getElementType() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ SPIR_FUNC
Used for SPIR non-kernel device functions.
@ C
The default llvm calling convention, compatible with C.
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty
LLVM_ABI bool getConstantStringInfo(const Value *V, StringRef &Str, bool TrimAtNul=true)
This function computes the length of a null-terminated C string pointed to by V.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool sortBlocks(Function &F)
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
@ Ref
The access may reference the value stored in memory.
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
void replace(R &&Range, const T &OldValue, const T &NewValue)
Provide wrappers to std::replace which take ranges instead of having to pass begin/end explicitly.
constexpr unsigned BitWidth
ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy
LLVM_ABI void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool to_integer(StringRef S, N &Num, unsigned Base=0)
Convert the string S to an integer of the specified type using the radix Base. If Base is 0,...
ModulePass * createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM)
LLVM_ABI void expandMemSetAsLoop(MemSetInst *MemSet)
Expand MemSet as a loop. MemSet is not deleted.
Implement std::hash so that hash_code can be used in STL containers.