[LV] Unify interleaved load handling for fixed and scalable VFs. nfc #146914
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-llvm-transforms Author: Mel Chen (Mel-Chen) ChangesThis patch modifies VPInterleaveRecipe::execute to handle both fixed and scalable VFs using a single loop. Full diff: https://github.com/llvm/llvm-project/pull/146914.diff 1 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 06511b61a67c3..2fc2447deb3fb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3442,7 +3442,6 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
VPValue *BlockInMask = getMask();
VPValue *Addr = getAddr();
Value *ResAddr = State.get(Addr, VPLane(0));
- Value *PoisonVec = PoisonValue::get(VecTy);
auto CreateGroupMask = [&BlockInMask, &State,
&InterleaveFactor](Value *MaskForGaps) -> Value * {
@@ -3481,6 +3480,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
Instruction *NewLoad;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
+ Value *PoisonVec = PoisonValue::get(VecTy);
NewLoad = State.Builder.CreateMaskedLoad(VecTy, ResAddr,
Group->getAlign(), GroupMask,
PoisonVec, "wide.masked.vec");
@@ -3490,57 +3490,39 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
Group->addMetadata(NewLoad);
ArrayRef<VPValue *> VPDefs = definedValues();
- const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
if (VecTy->isScalableTy()) {
// Scalable vectors cannot use arbitrary shufflevectors (only splats),
// so must use intrinsics to deinterleave.
assert(InterleaveFactor <= 8 &&
"Unsupported deinterleave factor for scalable vectors");
- Value *Deinterleave = State.Builder.CreateIntrinsic(
+ NewLoad = State.Builder.CreateIntrinsic(
getDeinterleaveIntrinsicID(InterleaveFactor), NewLoad->getType(),
NewLoad,
/*FMFSource=*/nullptr, "strided.vec");
+ }
- for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {
- Instruction *Member = Group->getMember(I);
- Value *StridedVec = State.Builder.CreateExtractValue(Deinterleave, I);
- if (!Member) {
- // This value is not needed as it's not used
- cast<Instruction>(StridedVec)->eraseFromParent();
- continue;
- }
- // If this member has different type, cast the result type.
- if (Member->getType() != ScalarTy) {
- VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
- StridedVec =
- createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL);
- }
-
- if (Group->isReverse())
- StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
-
- State.set(VPDefs[J], StridedVec);
- ++J;
- }
+ auto CreateStridedVector = [&InterleaveFactor, &State,
+ &NewLoad](unsigned Index) -> Value * {
+ assert(Index < InterleaveFactor && "Illegal group index");
+ if (State.VF.isScalable())
+ return State.Builder.CreateExtractValue(NewLoad, Index);
- return;
- }
- assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
+ // For fixed length VF, use shuffle to extract the sub-vectors from the
+ // wide load.
+ auto StrideMask =
+ createStrideMask(Index, InterleaveFactor, State.VF.getFixedValue());
+ return State.Builder.CreateShuffleVector(NewLoad, StrideMask,
+ "strided.vec");
+ };
- // For each member in the group, shuffle out the appropriate data from the
- // wide loads.
- unsigned J = 0;
- for (unsigned I = 0; I < InterleaveFactor; ++I) {
+ for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {
Instruction *Member = Group->getMember(I);
// Skip the gaps in the group.
if (!Member)
continue;
- auto StrideMask =
- createStrideMask(I, InterleaveFactor, State.VF.getFixedValue());
- Value *StridedVec =
- State.Builder.CreateShuffleVector(NewLoad, StrideMask, "strided.vec");
+ Value *StridedVec = CreateStridedVector(I);
// If this member has different type, cast the result type.
if (Member->getType() != ScalarTy) {
|
|
@llvm/pr-subscribers-vectorizers Author: Mel Chen (Mel-Chen) ChangesThis patch modifies VPInterleaveRecipe::execute to handle both fixed and scalable VFs using a single loop. Full diff: https://github.com/llvm/llvm-project/pull/146914.diff 1 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 06511b61a67c3..2fc2447deb3fb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3442,7 +3442,6 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
VPValue *BlockInMask = getMask();
VPValue *Addr = getAddr();
Value *ResAddr = State.get(Addr, VPLane(0));
- Value *PoisonVec = PoisonValue::get(VecTy);
auto CreateGroupMask = [&BlockInMask, &State,
&InterleaveFactor](Value *MaskForGaps) -> Value * {
@@ -3481,6 +3480,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
Instruction *NewLoad;
if (BlockInMask || MaskForGaps) {
Value *GroupMask = CreateGroupMask(MaskForGaps);
+ Value *PoisonVec = PoisonValue::get(VecTy);
NewLoad = State.Builder.CreateMaskedLoad(VecTy, ResAddr,
Group->getAlign(), GroupMask,
PoisonVec, "wide.masked.vec");
@@ -3490,57 +3490,39 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
Group->addMetadata(NewLoad);
ArrayRef<VPValue *> VPDefs = definedValues();
- const DataLayout &DL = State.CFG.PrevBB->getDataLayout();
if (VecTy->isScalableTy()) {
// Scalable vectors cannot use arbitrary shufflevectors (only splats),
// so must use intrinsics to deinterleave.
assert(InterleaveFactor <= 8 &&
"Unsupported deinterleave factor for scalable vectors");
- Value *Deinterleave = State.Builder.CreateIntrinsic(
+ NewLoad = State.Builder.CreateIntrinsic(
getDeinterleaveIntrinsicID(InterleaveFactor), NewLoad->getType(),
NewLoad,
/*FMFSource=*/nullptr, "strided.vec");
+ }
- for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {
- Instruction *Member = Group->getMember(I);
- Value *StridedVec = State.Builder.CreateExtractValue(Deinterleave, I);
- if (!Member) {
- // This value is not needed as it's not used
- cast<Instruction>(StridedVec)->eraseFromParent();
- continue;
- }
- // If this member has different type, cast the result type.
- if (Member->getType() != ScalarTy) {
- VectorType *OtherVTy = VectorType::get(Member->getType(), State.VF);
- StridedVec =
- createBitOrPointerCast(State.Builder, StridedVec, OtherVTy, DL);
- }
-
- if (Group->isReverse())
- StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
-
- State.set(VPDefs[J], StridedVec);
- ++J;
- }
+ auto CreateStridedVector = [&InterleaveFactor, &State,
+ &NewLoad](unsigned Index) -> Value * {
+ assert(Index < InterleaveFactor && "Illegal group index");
+ if (State.VF.isScalable())
+ return State.Builder.CreateExtractValue(NewLoad, Index);
- return;
- }
- assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
+ // For fixed length VF, use shuffle to extract the sub-vectors from the
+ // wide load.
+ auto StrideMask =
+ createStrideMask(Index, InterleaveFactor, State.VF.getFixedValue());
+ return State.Builder.CreateShuffleVector(NewLoad, StrideMask,
+ "strided.vec");
+ };
- // For each member in the group, shuffle out the appropriate data from the
- // wide loads.
- unsigned J = 0;
- for (unsigned I = 0; I < InterleaveFactor; ++I) {
+ for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {
Instruction *Member = Group->getMember(I);
// Skip the gaps in the group.
if (!Member)
continue;
- auto StrideMask =
- createStrideMask(I, InterleaveFactor, State.VF.getFixedValue());
- Value *StridedVec =
- State.Builder.CreateShuffleVector(NewLoad, StrideMask, "strided.vec");
+ Value *StridedVec = CreateStridedVector(I);
// If this member has different type, cast the result type.
if (Member->getType() != ScalarTy) {
|
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This patch modifies VPInterleaveRecipe::execute to handle both fixed and scalable VFs using a single loop.