This file implements explicit unrolling for VPlans. More...

#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Intrinsics.h"

Functions
static void	addStartIndexForScalarSteps (VPScalarIVStepsRecipe *Steps, unsigned Part, VPlan &Plan, VPTypeAnalysis &TypeInfo)
static void	addLaneToStartIndex (VPScalarIVStepsRecipe Steps, unsigned Lane, VPlan &Plan, VPRecipeBase InsertPt)
	Add a lane offset to the start index of `Steps`.
static VPValue *	cloneForLane (VPlan &Plan, VPBuilder &Builder, Type IdxTy, VPSingleDefRecipe DefR, VPLane Lane, const DenseMap< VPValue , SmallVector< VPValue > > &Def2LaneDefs)
	Create a single-scalar clone of `DefR` (must be a VPReplicateRecipe, VPInstruction or VPScalarIVStepsRecipe) for lane `Lane`.
static void	convertRecipesInRegionBlocksToSingleScalar (VPlan &Plan, Type IdxTy, VPBlockBase Entry, ElementCount VF)
	Convert recipes in region blocks to operate on a single lane 0.
static void	processLaneForReplicateRegion (VPlan &Plan, Type IdxTy, unsigned Lane, VPBasicBlock OldEntry, VPBasicBlock *NewEntry)
	Update recipes in the cloned blocks rooted at `NewEntry` to match `Lane`, using the original blocks rooted at `OldEntry` as reference.
static void	dissolveReplicateRegion (VPRegionBlock Region, ElementCount VF, VPlan &Plan, Type IdxTy)
	Dissolve a single replicate region by replicating its blocks for each lane of `VF`.
static void	replicateReplicateRegionsByVF (VPlan &Plan, ElementCount VF, Type *IdxTy)
	Collect and dissolve all replicate regions in the vector loop, replicating their blocks and recipes for each lane of `VF`.

Detailed Description

This file implements explicit unrolling for VPlans.

Definition in file VPlanUnroll.cpp.

Function Documentation

◆ addLaneToStartIndex()

void addLaneToStartIndex	(	VPScalarIVStepsRecipe *	Steps,
		unsigned	Lane,
		VPlan &	Plan,
		VPRecipeBase *	InsertPt )

static

Add a lane offset to the start index of Steps.

Definition at line 553 of file VPlanUnroll.cpp.

References assert(), llvm::VPlan::getConstantInt(), llvm::VPScalarIVStepsRecipe::getInductionOpcode(), llvm::VPUser::getOperand(), llvm::VPlan::getOrAddLiveIn(), llvm::Type::getScalarSizeInBits(), llvm::VPScalarIVStepsRecipe::getStartIndex(), llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isFloatingPointTy(), and llvm::VPScalarIVStepsRecipe::setStartIndex().

Referenced by cloneForLane(), and processLaneForReplicateRegion().

◆ addStartIndexForScalarSteps()

void addStartIndexForScalarSteps	(	VPScalarIVStepsRecipe *	Steps,
		unsigned	Part,
		VPlan &	Plan,
		VPTypeAnalysis &	TypeInfo )

static

Definition at line 127 of file VPlanUnroll.cpp.

References llvm::IntegerType::get(), llvm::VPlan::getConstantInt(), llvm::Type::getContext(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPUser::getOperand(), llvm::Type::getScalarSizeInBits(), llvm::VPScalarIVStepsRecipe::getVFValue(), llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isFloatingPointTy(), and llvm::VPScalarIVStepsRecipe::setStartIndex().

◆ cloneForLane()

VPValue * cloneForLane	(	VPlan &	Plan,
		VPBuilder &	Builder,
		Type *	IdxTy,
		VPSingleDefRecipe *	DefR,
		VPLane	Lane,
		const DenseMap< VPValue , SmallVector< VPValue > > &	Def2LaneDefs )

static

Create a single-scalar clone of DefR (must be a VPReplicateRecipe, VPInstruction or VPScalarIVStepsRecipe) for lane Lane.

Use Def2LaneDefs to look up scalar definitions for operands of \DefR.

Definition at line 592 of file VPlanUnroll.cpp.

References addLaneToStartIndex(), assert(), llvm::cast(), llvm::VPSingleDefRecipe::clone(), llvm::dyn_cast(), llvm::enumerate(), llvm::VPInstruction::ExtractLastLane, llvm::VPInstruction::ExtractLastPart, llvm::VPlan::getConstantInt(), llvm::VPLane::getKind(), llvm::VPLane::getKnownLane(), llvm::isa(), llvm::vputils::isSingleScalar(), llvm::VPlanPatternMatch::m_BuildVector(), llvm::VPlanPatternMatch::m_VPInstruction(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPUser::operands(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::VPLane::ScalableLast.

Referenced by llvm::VPlanTransforms::replicateByVF().

◆ convertRecipesInRegionBlocksToSingleScalar()

void convertRecipesInRegionBlocksToSingleScalar	(	VPlan &	Plan,
		Type *	IdxTy,
		VPBlockBase *	Entry,
		ElementCount	VF )

static

Convert recipes in region blocks to operate on a single lane 0.

VPReplicateRecipes are converted to single-scalar ones, branch-on-mask is converted into BranchOnCond and extracts are created as needed.

Definition at line 672 of file VPlanUnroll.cpp.

References assert(), llvm::VPInstruction::BranchOnCond, llvm::cast(), llvm::dyn_cast(), llvm::enumerate(), llvm::PoisonValue::get(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getZero(), I, llvm::VPTypeAnalysis::inferScalarType(), llvm::isa(), llvm::isa_and_present(), llvm::ElementCount::isScalar(), llvm::vputils::isSingleScalar(), llvm::VPlanPatternMatch::m_ExtractElement(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::VPValue::replaceAllUsesWith(), and llvm::vp_depth_first_shallow().

Referenced by dissolveReplicateRegion().

◆ dissolveReplicateRegion()

void dissolveReplicateRegion	(	VPRegionBlock *	Region,
		ElementCount	VF,
		VPlan &	Plan,
		Type *	IdxTy )

static

Dissolve a single replicate region by replicating its blocks for each lane of VF.

The region is disconnected, its blocks are reparented, cloned for each lane, and reconnected in sequence.

Definition at line 768 of file VPlanUnroll.cpp.

References assert(), llvm::cast(), llvm::VPBlockUtils::cloneFrom(), llvm::VPBlockUtils::connectBlocks(), convertRecipesInRegionBlocksToSingleScalar(), llvm::VPBlockUtils::disconnectBlocks(), llvm::RegionBase< Tr >::getEntry(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::RegionBase< Tr >::getParent(), processLaneForReplicateRegion(), llvm::VPBlockBase::setParent(), llvm::Successor, and llvm::vp_depth_first_shallow().

Referenced by replicateReplicateRegionsByVF().

◆ processLaneForReplicateRegion()

void processLaneForReplicateRegion	(	VPlan &	Plan,
		Type *	IdxTy,
		unsigned	Lane,
		VPBasicBlock *	OldEntry,
		VPBasicBlock *	NewEntry )

static

Update recipes in the cloned blocks rooted at NewEntry to match Lane, using the original blocks rooted at OldEntry as reference.

Definition at line 736 of file VPlanUnroll.cpp.

References addLaneToStartIndex(), llvm::cast(), llvm::dyn_cast(), llvm::enumerate(), llvm::VPlan::getConstantInt(), I, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::VPlanPatternMatch::m_ExtractElement(), llvm::VPlanPatternMatch::m_VPValue(), llvm::MIPatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), llvm::vp_depth_first_shallow(), and llvm::zip_equal().

Referenced by dissolveReplicateRegion().

◆ replicateReplicateRegionsByVF()

void replicateReplicateRegionsByVF	(	VPlan &	Plan,
		ElementCount	VF,
		Type *	IdxTy )

static

Collect and dissolve all replicate regions in the vector loop, replicating their blocks and recipes for each lane of VF.

Definition at line 815 of file VPlanUnroll.cpp.

References assert(), llvm::VPBlockUtils::blocksOnly(), dissolveReplicateRegion(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::VPRegionBlock::getEntry(), llvm::VPlan::getVectorLoopRegion(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable(), llvm::ElementCount::isScalar(), llvm::VPlanTransforms::mergeBlocksIntoPredecessors(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::vp_depth_first_shallow().

Referenced by llvm::VPlanTransforms::replicateByVF().

Functions

Detailed Description

Function Documentation

◆ addLaneToStartIndex()

◆ addStartIndexForScalarSteps()

◆ cloneForLane()

◆ convertRecipesInRegionBlocksToSingleScalar()

◆ dissolveReplicateRegion()

◆ processLaneForReplicateRegion()

◆ replicateReplicateRegionsByVF()