@@ -3445,7 +3445,6 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
3445
3445
VPValue *BlockInMask = getMask ();
3446
3446
VPValue *Addr = getAddr ();
3447
3447
Value *ResAddr = State.get (Addr, VPLane (0 ));
3448
- Value *PoisonVec = PoisonValue::get (VecTy);
3449
3448
3450
3449
auto CreateGroupMask = [&BlockInMask, &State,
3451
3450
&InterleaveFactor](Value *MaskForGaps) -> Value * {
@@ -3484,6 +3483,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
3484
3483
Instruction *NewLoad;
3485
3484
if (BlockInMask || MaskForGaps) {
3486
3485
Value *GroupMask = CreateGroupMask (MaskForGaps);
3486
+ Value *PoisonVec = PoisonValue::get (VecTy);
3487
3487
NewLoad = State.Builder .CreateMaskedLoad (VecTy, ResAddr,
3488
3488
Group->getAlign (), GroupMask,
3489
3489
PoisonVec, " wide.masked.vec" );
@@ -3493,57 +3493,39 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
3493
3493
Group->addMetadata (NewLoad);
3494
3494
3495
3495
ArrayRef<VPValue *> VPDefs = definedValues ();
3496
- const DataLayout &DL = State.CFG .PrevBB ->getDataLayout ();
3497
3496
if (VecTy->isScalableTy ()) {
3498
3497
// Scalable vectors cannot use arbitrary shufflevectors (only splats),
3499
3498
// so must use intrinsics to deinterleave.
3500
3499
assert (InterleaveFactor <= 8 &&
3501
3500
" Unsupported deinterleave factor for scalable vectors" );
3502
- Value *Deinterleave = State.Builder .CreateIntrinsic (
3501
+ NewLoad = State.Builder .CreateIntrinsic (
3503
3502
getDeinterleaveIntrinsicID (InterleaveFactor), NewLoad->getType (),
3504
3503
NewLoad,
3505
3504
/* FMFSource=*/ nullptr , " strided.vec" );
3505
+ }
3506
3506
3507
- for (unsigned I = 0 , J = 0 ; I < InterleaveFactor; ++I) {
3508
- Instruction *Member = Group->getMember (I);
3509
- Value *StridedVec = State.Builder .CreateExtractValue (Deinterleave, I);
3510
- if (!Member) {
3511
- // This value is not needed as it's not used
3512
- cast<Instruction>(StridedVec)->eraseFromParent ();
3513
- continue ;
3514
- }
3515
- // If this member has different type, cast the result type.
3516
- if (Member->getType () != ScalarTy) {
3517
- VectorType *OtherVTy = VectorType::get (Member->getType (), State.VF );
3518
- StridedVec =
3519
- createBitOrPointerCast (State.Builder , StridedVec, OtherVTy, DL);
3520
- }
3521
-
3522
- if (Group->isReverse ())
3523
- StridedVec = State.Builder .CreateVectorReverse (StridedVec, " reverse" );
3524
-
3525
- State.set (VPDefs[J], StridedVec);
3526
- ++J;
3527
- }
3507
+ auto CreateStridedVector = [&InterleaveFactor, &State,
3508
+ &NewLoad](unsigned Index) -> Value * {
3509
+ assert (Index < InterleaveFactor && " Illegal group index" );
3510
+ if (State.VF .isScalable ())
3511
+ return State.Builder .CreateExtractValue (NewLoad, Index);
3528
3512
3529
- return ;
3530
- }
3531
- assert (!State.VF .isScalable () && " VF is assumed to be non scalable." );
3513
+ // For fixed length VF, use shuffle to extract the sub-vectors from the
3514
+ // wide load.
3515
+ auto StrideMask =
3516
+ createStrideMask (Index, InterleaveFactor, State.VF .getFixedValue ());
3517
+ return State.Builder .CreateShuffleVector (NewLoad, StrideMask,
3518
+ " strided.vec" );
3519
+ };
3532
3520
3533
- // For each member in the group, shuffle out the appropriate data from the
3534
- // wide loads.
3535
- unsigned J = 0 ;
3536
- for (unsigned I = 0 ; I < InterleaveFactor; ++I) {
3521
+ for (unsigned I = 0 , J = 0 ; I < InterleaveFactor; ++I) {
3537
3522
Instruction *Member = Group->getMember (I);
3538
3523
3539
3524
// Skip the gaps in the group.
3540
3525
if (!Member)
3541
3526
continue ;
3542
3527
3543
- auto StrideMask =
3544
- createStrideMask (I, InterleaveFactor, State.VF .getFixedValue ());
3545
- Value *StridedVec =
3546
- State.Builder .CreateShuffleVector (NewLoad, StrideMask, " strided.vec" );
3528
+ Value *StridedVec = CreateStridedVector (I);
3547
3529
3548
3530
// If this member has different type, cast the result type.
3549
3531
if (Member->getType () != ScalarTy) {
0 commit comments