[AIEX] Reorder PostInc and PreInc Memory Instr

Author: F-Stuckmann

llvm/lib/Target/AIE/AIEGlobalCombiner.cpp

@@ -69,9 +69,12 @@ AIEGlobalCombiner::findBeneficialCombiners() {
     CombineCandidates.filterOut(FixedCombiners);
     CombineCandidates.filterOut(FoundCombiners);
 
-    for (auto *Combiner :
-         CombineCandidates.searchCombinerSet(OwnedCombineCandidates))
-      FoundCombiners.push_back(Combiner);
+    auto Combiners =
+        CombineCandidates.searchCombinerSet(OwnedCombineCandidates);
+    reorderCombinerInsertions(Combiners);
+
+    FoundCombiners.insert(FoundCombiners.end(), Combiners.begin(),
+                          Combiners.end());
   }
 
   LLVM_DEBUG(dbgs() << "[Global Combiner] Found " << FoundCombiners.size()
@@ -80,6 +83,27 @@ AIEGlobalCombiner::findBeneficialCombiners() {
   return FoundCombiners;
 }
 
+void AIEGlobalCombiner::reorderCombinerInsertions(
+    std::vector<GenericCombiner *> &Combiners) const {
+  for (auto *Combiner : Combiners) {
+    if (!Combiner->canReorder())
+      continue;
+
+    auto It = std::find_if(Combiners.rbegin(), Combiners.rend(),
+                           [Combiner](GenericCombiner *Candidate) {
+                             return Combiner->isReorderCandidate(Candidate);
+                           });
+
+    if (It == Combiners.rend())
+      continue;
+
+    auto *BestCandidate = *It;
+    Combiner->copyInsertionPoint(BestCandidate);
+    LLVM_DEBUG(dbgs() << "Reordering \n"; Combiner->dumpFull();
+               BestCandidate->dumpFull(););
+  }
+}
+
 void AIEGlobalCombiner::calculateCombineCandidates(
     SUnit &CombineRoot, const GenericCombiner *Combiner) {
   assert(MDT);
@@ -198,7 +222,7 @@ std::vector<CombineCandidates> AIEGlobalCombiner::getCombineCandidates(
 
 // -------------------------- CombineCandidates ------------------------------//
 
-std::vector<const GenericCombiner *> CombineCandidates::searchCombinerSet(
+std::vector<GenericCombiner *> CombineCandidates::searchCombinerSet(
     const std::vector<std::unique_ptr<GenericCombiner>>
         &OwnedCombineCandidates) {
   if (Combiners.empty())
@@ -299,7 +323,7 @@ std::vector<const GenericCombiner *> CombineCandidates::searchCombinerSet(
   LLVM_DEBUG(dbgs() << "Search Result " << BestSolution.getGain() << "\n");
 
   // Save best Candidate to FixedCombiners
-  std::vector<const GenericCombiner *> Result;
+  std::vector<GenericCombiner *> Result;
   BitVector CombinerBitVec = BestSolution.getCombinersBitVector();
   for (int Idx = CombinerBitVec.find_first(); Idx != -1;
        Idx = CombinerBitVec.find_next(Idx)) {
@@ -517,6 +541,16 @@ void GenericCombiner::setGlobalID(unsigned GlobalID) {
   this->GlobalID = GlobalID;
 }
 
+bool GenericCombiner::isReorderCandidate(
+    const GenericCombiner *Candidate) const {
+  return false;
+}
+
+void GenericCombiner::copyInsertionPoint(const GenericCombiner *Candidate) {
+  CombinerData.InsertionPoint = Candidate->CombinerData.InsertionPoint;
+  InsertionPointNodeNum = Candidate->InsertionPointNodeNum;
+}
+
 /// \return whether a Combiner is used after a Remove-Combiner, that
 /// are part of the same Cluster. The Ordering of the Combiners \p A and \p B is
 /// irrelevant.

llvm/lib/Target/AIE/AIEGlobalCombiner.h

@@ -256,6 +256,16 @@ class GenericCombiner {
 
   /// Set unique Identifier for this Combiner to \p GlobalID
   void setGlobalID(unsigned GlobalID);
+
+  /// \return whether this Combiner could be moved before another Combiner
+  virtual bool canReorder() const = 0;
+
+  /// \return whether \p Candidate is a ReorderCandidate, i.e. if this combiner
+  /// can be inserted right before \p Candidate
+  virtual bool isReorderCandidate(const GenericCombiner *Candidate) const;
+
+  /// Set InsertionPoint of this combiner to the same as \p Candidate
+  void copyInsertionPoint(const GenericCombiner *Candidate);
 };
 
 raw_ostream &operator<<(raw_ostream &OS, const GenericCombiner &Val);
@@ -338,7 +348,7 @@ class CombineCandidates {
 
   /// \return Combiners from \p OwnedCombineCandidates that maximize the gain
   /// when applied
-  std::vector<const GenericCombiner *>
+  std::vector<GenericCombiner *>
   searchCombinerSet(const std::vector<std::unique_ptr<GenericCombiner>>
                         &OwnedCombineCandidates);
 
@@ -396,6 +406,9 @@ class AIEGlobalCombiner {
 
   void calculateCombinerConflicts();
 
+  void
+  reorderCombinerInsertions(std::vector<GenericCombiner *> &Combiners) const;
+
   /// \return CombineCandidates sorted by highest potential gain
   std::vector<CombineCandidates> getCombineCandidates(
       std::map<Register, CombineCandidates> ClusteredCombiners);

llvm/lib/Target/AIE/AIEGlobalCombinerPtrMods.cpp

@@ -315,7 +315,25 @@ std::unique_ptr<GenericCombiner> OffsetCombiner::clone() const {
 
 std::optional<std::pair<std::vector<SUnit *>, std::vector<SUnit *>>>
 OffsetCombiner::getInstructionsToMove(const AIE::DataDependenceHelper &DAG) {
-  return {{/*MoveUp*/ {}, /*MoveDown*/ {}}};
+  auto *PtrAdd = getPtrInc();
+  if (!getImm(*PtrAdd, *MRI)) {
+    /// Offset is not an immediate and the OffsetCombiner is not eligible for
+    /// reordering.
+    // Since the Offset already dominates the MemoryInstruction (where the
+    // insertion happens), no checks have to be performed.
+    return {{/*MoveUp=*/{}, /*MoveDown=*/{}}};
+  }
+
+  auto *SUnitPtrAdd = DAG.getSUnit(PtrAdd);
+  if (!SUnitPtrAdd) {
+    /// PtrAdd is an Immediate but it is outside of the MBB, so it already
+    /// dominates the MemoryInstruction. No checks have to be performed.
+    return {{/*MoveUp=*/{}, /*MoveDown=*/{}}};
+  }
+
+  /// Immediate Offset can be a reordering Candidate. Therefore, track Immediate
+  /// Offset, so it can be moved in case of a reordering.
+  return {{/*MoveUp=*/{SUnitPtrAdd}, /*MoveDown=*/{}}};
 }
 
 void OffsetCombiner::adjustGain(const MachineDominatorTree &MDT) {
@@ -335,7 +353,7 @@ void OffsetCombiner::adjustGain(const MachineDominatorTree &MDT) {
     Gain.setPtrMod(0);
   }
 
-  std::optional<APInt> ImmOffset = getImm(*PtrAdd, *MRI);
+  ImmOffset = getImm(*PtrAdd, *MRI);
   if (!ImmOffset)
     return;
 
@@ -363,6 +381,58 @@ std::optional<unsigned> OffsetCombiner::getOpCode(MachineInstr *PtrInc,
   return TII->getOffsetMemOpcode(MemI->getOpcode());
 }
 
+bool OffsetCombiner::canReorder() const { return ImmOffset.has_value(); }
+
+bool OffsetCombiner::isReorderCandidate(
+    const GenericCombiner *PostIncCombiner) const {
+  auto GetInputPtr = [&](const MachineInstr *PtrMod) {
+    auto InputPtrIdx = PtrModSupport.getInputPtrIdx(*PtrMod);
+    assert(InputPtrIdx);
+    return PtrMod->getOperand(*InputPtrIdx);
+  };
+
+  const PointerModifierCombiner *PtrModCombiner =
+      static_cast<const PointerModifierCombiner *>(PostIncCombiner);
+  if (!PtrModCombiner->isPostInc())
+    return false;
+
+  // only allow loads to be reordered
+  if (getMemI()->mayStore() || PtrModCombiner->getMemI()->mayStore())
+    return false;
+
+  // Same MBB check
+  auto *PtrAdd = getPtrInc();
+  auto *PostIncPtrMod = PtrModCombiner->getPtrInc();
+  if (PtrAdd->getParent() != PostIncPtrMod->getParent())
+    return false;
+
+  // Same Input Ptr Check
+  auto InputPtr = GetInputPtr(PtrAdd);
+  auto PostIncInputPtr = GetInputPtr(PostIncPtrMod);
+  if (!InputPtr.isIdenticalTo(PostIncInputPtr))
+    return false;
+
+  // Check if Store Instruction of Offset dominates PostInc
+  auto *MemI = getMemI();
+  if (MemI->mayStore()) {
+    auto Source = MemI->getOperand(0);
+    assert(Source.isReg());
+    auto *DefSource = MRI->getUniqueVRegDef(Source.getReg());
+    if (!DefSource)
+      return false;
+    auto *DefSUnit = DAG->getSUnit(DefSource);
+    if (DefSUnit &&
+        DefSUnit->NodeNum > PostIncCombiner->InsertionPointNodeNum) {
+      // Source of Offset-Store would be after the new InsertionPoint and thus
+      // generate invalid mir
+      return false;
+    }
+  }
+
+  // OffsetCombiner occurs after PostIncCombiner
+  return InsertionPointNodeNum > PostIncCombiner->InsertionPointNodeNum;
+}
+
 // -------------------------- PostIncCombiner --------------------------------//
 
 bool PostIncCombiner::isCombineCandidate(MachineInstr &MemI,

llvm/lib/Target/AIE/AIEGlobalCombinerPtrMods.h

@@ -118,9 +118,13 @@ class PointerModifierCombiner : public GenericCombiner {
 
   /// \return whether Opcode can be set
   bool tryToSetCombinedOpCode() override;
+
+  virtual bool isPostInc() const = 0;
 };
 
 class OffsetCombiner : public PointerModifierCombiner {
+  std::optional<APInt> ImmOffset;
+
 protected:
   std::optional<unsigned> getOpCode(MachineInstr *PtrInc,
                                     MachineInstr *MemI) const override;
@@ -141,6 +145,12 @@ class OffsetCombiner : public PointerModifierCombiner {
 
   std::optional<std::pair<std::vector<SUnit *>, std::vector<SUnit *>>>
   getInstructionsToMove(const AIE::DataDependenceHelper &DAG) override;
+
+  bool isReorderCandidate(const GenericCombiner *Candidate) const override;
+
+  bool canReorder() const override;
+
+  bool isPostInc() const override { return false; }
 };
 
 class PostIncCombiner : public PointerModifierCombiner {
@@ -176,6 +186,14 @@ class PostIncCombiner : public PointerModifierCombiner {
   void adjustGain(const MachineDominatorTree &MDT) override;
 
   std::vector<MachineInstr *> getPtrInstrs(MachineInstr *MI) const override;
+
+  bool isReorderCandidate(const GenericCombiner *Candidate) const override {
+    return false;
+  }
+
+  bool canReorder() const override { return false; }
+
+  bool isPostInc() const override { return true; }
 };
 
 } // namespace llvm::AIE

llvm/lib/Target/AIE/AIEPtrModOptimizer.cpp

@@ -167,6 +167,9 @@ void FoundCombiners::remapCombiner(AIE::Combiner &Combiner) const {
       getRemappedInstrs(Combiner.DelayInstrToInsertionPoint);
   Combiner.DelayInstrPastInsertionPoint =
       getRemappedInstrs(Combiner.DelayInstrPastInsertionPoint);
+
+  std::vector<MachineInstr *> InsertionPointVec = {Combiner.InsertionPoint};
+  Combiner.InsertionPoint = getRemappedInstrs(InsertionPointVec)[0];
 }
 
 const std::map<MachineInstr *, AIE::Combiner> &

llvm/test/CodeGen/AIE/aie2p/GlobalIsel/global-combiners/gemm.mir

@@ -40,9 +40,9 @@ body:             |
   ; CHECK-NEXT:   [[PHI1:%[0-9]+]]:_(p0) = G_PHI [[COPY]](p0), %bb.0, %9(p0), %bb.1
   ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI]], [[C2]]
   ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(sgt), [[ADD]](s32), [[C]]
-  ; CHECK-NEXT:   [[AIE_POSTINC_3D_LOAD:%[0-9]+]]:_(<32 x s16>), [[AIE_POSTINC_3D_LOAD1:%[0-9]+]]:_(p0), [[AIE_POSTINC_3D_LOAD2:%[0-9]+]]:_(s20), [[AIE_POSTINC_3D_LOAD3:%[0-9]+]]:_ = G_AIE_POSTINC_3D_LOAD [[PHI1]], [[C1]], [[C1]], [[C1]], [[C1]], [[C1]], [[C1]], [[C1]] :: (load (<32 x s16>))
   ; CHECK-NEXT:   [[C4:%[0-9]+]]:_(s20) = G_CONSTANT i20 64
   ; CHECK-NEXT:   [[AIE_OFFSET_LOAD:%[0-9]+]]:_(<32 x s16>) = G_AIE_OFFSET_LOAD [[PHI1]](p0), [[C4]](s20) :: (load (<32 x s16>))
+  ; CHECK-NEXT:   [[AIE_POSTINC_3D_LOAD:%[0-9]+]]:_(<32 x s16>), [[AIE_POSTINC_3D_LOAD1:%[0-9]+]]:_(p0), [[AIE_POSTINC_3D_LOAD2:%[0-9]+]]:_(s20), [[AIE_POSTINC_3D_LOAD3:%[0-9]+]]:_ = G_AIE_POSTINC_3D_LOAD [[PHI1]], [[C1]], [[C1]], [[C1]], [[C1]], [[C1]], [[C1]], [[C1]] :: (load (<32 x s16>))
   ; CHECK-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[PHI1]], %configZero(s20)
   ; CHECK-NEXT:   [[C5:%[0-9]+]]:_(s20) = G_CONSTANT i20 0
   ; CHECK-NEXT:   [[AIE_OFFSET_LOAD1:%[0-9]+]]:_(<32 x s16>) = G_AIE_OFFSET_LOAD [[PTR_ADD]](p0), [[C5]](s20) :: (load (<32 x s16>))

llvm/test/CodeGen/AIE/aie2p/GlobalIsel/global-combiners/post-inc-eagerness.mir

@@ -17,6 +17,7 @@
 # increasing register pressure.
 # FIXME: reorder instructions
 
+# Do not reorder Store Instructions.
 ---
 name:            post-inc-reg-pressure-store
 legalized:       true
@@ -46,6 +47,7 @@ body:             |
     PseudoRET implicit $lr, implicit %3
 ...
 
+# Reorder Load Instructions.
 # similar example as above, but with load instructions
 ---
 name:            post-inc-reg-pressure-load
@@ -60,9 +62,9 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $p0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s20) = G_CONSTANT i20 64
-    ; CHECK-NEXT: %lZero:_(s32), %7:_(p0), %8:_(s20), %9:_ = G_AIE_POSTINC_3D_LOAD [[COPY]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]] :: (load (s32))
     ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s20) = G_CONSTANT i20 8
     ; CHECK-NEXT: %lOne:_(s32) = G_AIE_OFFSET_LOAD [[COPY]](p0), [[C1]](s20) :: (load (s32))
+    ; CHECK-NEXT: %lZero:_(s32), %7:_(p0), %8:_(s20), %9:_ = G_AIE_POSTINC_3D_LOAD [[COPY]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]] :: (load (s32))
     ; CHECK-NEXT: PseudoRET implicit $lr, implicit %7(p0), implicit %lZero(s32), implicit %lOne(s32)
     %0:_(p0) = COPY $p0
     %1:_(s20) = G_CONSTANT i20 64

llvm/test/CodeGen/AIE/aie2p/GlobalIsel/global-combiners/reorder-Mem-Instrs.mir

@@ -28,8 +28,8 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $p0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s20) = G_CONSTANT i20 64
-    ; CHECK-NEXT: %sZero:_(s32), %6:_(p0), %7:_(s20), %8:_ = G_AIE_POSTINC_3D_LOAD [[COPY]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]] :: (load (s32))
     ; CHECK-NEXT: %sOne:_(s32) = G_AIE_OFFSET_LOAD [[COPY]](p0), [[C]](s20) :: (load (s32))
+    ; CHECK-NEXT: %sZero:_(s32), %6:_(p0), %7:_(s20), %8:_ = G_AIE_POSTINC_3D_LOAD [[COPY]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]] :: (load (s32))
     ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = nsw G_ADD %sZero, %sOne
     ; CHECK-NEXT: G_STORE [[ADD]](s32), %6(p0) :: (store (s32))
     ; CHECK-NEXT: PseudoRET implicit $lr

llvm/test/CodeGen/AIE/aie2p/GlobalIsel/global-combiners/user-intrinsics.mir

@@ -29,10 +29,10 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $p0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $p1
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s20) = G_CONSTANT i20 8
-    ; CHECK-NEXT: [[AIE_POSTINC_3D_LOAD:%[0-9]+]]:_(s32), [[AIE_POSTINC_3D_LOAD1:%[0-9]+]]:_(p0), [[AIE_POSTINC_3D_LOAD2:%[0-9]+]]:_(s20), [[AIE_POSTINC_3D_LOAD3:%[0-9]+]]:_ = G_AIE_POSTINC_3D_LOAD [[COPY]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]] :: (load (s32))
     ; CHECK-NEXT: [[AIE_OFFSET_LOAD:%[0-9]+]]:_(s32) = G_AIE_OFFSET_LOAD [[COPY]](p0), [[C]](s20) :: (load (s32))
-    ; CHECK-NEXT: [[AIE_POSTINC_3D_LOAD4:%[0-9]+]]:_(s32), [[AIE_POSTINC_3D_LOAD5:%[0-9]+]]:_(p0), [[AIE_POSTINC_3D_LOAD6:%[0-9]+]]:_(s20), [[AIE_POSTINC_3D_LOAD7:%[0-9]+]]:_ = G_AIE_POSTINC_3D_LOAD [[COPY1]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]] :: (load (s32))
+    ; CHECK-NEXT: [[AIE_POSTINC_3D_LOAD:%[0-9]+]]:_(s32), [[AIE_POSTINC_3D_LOAD1:%[0-9]+]]:_(p0), [[AIE_POSTINC_3D_LOAD2:%[0-9]+]]:_(s20), [[AIE_POSTINC_3D_LOAD3:%[0-9]+]]:_ = G_AIE_POSTINC_3D_LOAD [[COPY]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]] :: (load (s32))
     ; CHECK-NEXT: [[AIE_OFFSET_LOAD1:%[0-9]+]]:_(s32) = G_AIE_OFFSET_LOAD [[COPY1]](p0), [[C]](s20) :: (load (s32))
+    ; CHECK-NEXT: [[AIE_POSTINC_3D_LOAD4:%[0-9]+]]:_(s32), [[AIE_POSTINC_3D_LOAD5:%[0-9]+]]:_(p0), [[AIE_POSTINC_3D_LOAD6:%[0-9]+]]:_(s20), [[AIE_POSTINC_3D_LOAD7:%[0-9]+]]:_ = G_AIE_POSTINC_3D_LOAD [[COPY1]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]], [[C]] :: (load (s32))
     ; CHECK-NEXT: PseudoRET implicit $lr, implicit [[AIE_POSTINC_3D_LOAD]](s32), implicit [[AIE_OFFSET_LOAD]](s32), implicit [[AIE_POSTINC_3D_LOAD4]](s32), implicit [[AIE_OFFSET_LOAD1]](s32), implicit [[AIE_POSTINC_3D_LOAD1]](p0), implicit [[AIE_POSTINC_3D_LOAD5]](p0)
     %0:_(p0) = COPY $p0
     %1:_(p0) = COPY $p1