[RISCV] Support CoreV SIMD intrinsics (plctlab#46)

Co-authored-by: melonedo <[email protected]>

llvm/include/llvm/IR/IntrinsicsRISCV.td

@@ -1582,3 +1582,129 @@ def int_riscv_sm4ed      : ScalarCryptoByteSelectAny;
 def int_riscv_sm3p0      : ScalarCryptoGprIntrinsicAny;
 def int_riscv_sm3p1      : ScalarCryptoGprIntrinsicAny;
 } // TargetPrefix = "riscv"
+
+class ScalarCoreVSimdGprIntrinsic
+    : Intrinsic<[llvm_i32_ty], [llvm_i32_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable]>;
+
+class ScalarCoreVSimdGprGprIntrinsic
+    : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable]>;
+
+class ScalarCoreVSimdGprImmIntrinsic
+    : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable, ImmArg<ArgIndex<1>>]>;
+
+class ScalarCoreVSimdGprGprGprIntrinsic
+    : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable]>;
+
+class ScalarCoreVSimdGprGprImmIntrinsic
+    : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable, ImmArg<ArgIndex<2>>]>;
+
+class ScalarCoreVSimdGprGprGprImmIntrinsic
+    : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+                [IntrNoMem, IntrWillReturn, IntrSpeculatable, ImmArg<ArgIndex<3>>]>;
+
+multiclass ScalarCoreVSimdGprIntrinsicHB {
+  def int_riscv_cv_simd_ # NAME # _h : ScalarCoreVSimdGprIntrinsic;
+  def int_riscv_cv_simd_ # NAME # _b : ScalarCoreVSimdGprIntrinsic;
+}
+
+multiclass ScalarCoreVSimdGprGprIntrinsicHB {
+  def int_riscv_cv_simd_ # NAME # _h : ScalarCoreVSimdGprGprIntrinsic;
+  def int_riscv_cv_simd_ # NAME # _b : ScalarCoreVSimdGprGprIntrinsic;
+}
+
+multiclass ScalarCoreVSimdGprGprGprIntrinsicHB {
+  def int_riscv_cv_simd_ # NAME # _h : ScalarCoreVSimdGprGprGprIntrinsic;
+  def int_riscv_cv_simd_ # NAME # _b : ScalarCoreVSimdGprGprGprIntrinsic;
+}
+
+multiclass ScalarCoreVSimdGprGprIntrinsicDiv {
+  def int_riscv_cv_simd_ # NAME # _div2 : ScalarCoreVSimdGprGprIntrinsic;
+  def int_riscv_cv_simd_ # NAME # _div4 : ScalarCoreVSimdGprGprIntrinsic;
+  def int_riscv_cv_simd_ # NAME # _div8 : ScalarCoreVSimdGprGprIntrinsic;
+}
+
+multiclass ScalarCoreVSimdGprImmIntrinsicHB {
+  def int_riscv_cv_simd_ # NAME # _h : ScalarCoreVSimdGprImmIntrinsic;
+  def int_riscv_cv_simd_ # NAME # _b : ScalarCoreVSimdGprImmIntrinsic;
+}
+
+multiclass CoreVSimdBinary <bit exclude_h = false> {
+  if exclude_h then {
+    def int_riscv_cv_simd_ # NAME # _b : ScalarCoreVSimdGprGprIntrinsic;
+  } else {
+    defm NAME : ScalarCoreVSimdGprGprIntrinsicHB;
+  }
+  defm NAME # _sc : ScalarCoreVSimdGprGprIntrinsicHB;
+}
+
+multiclass CoreVSimdTernary {
+  defm NAME : ScalarCoreVSimdGprGprGprIntrinsicHB;
+  defm NAME # _sc : ScalarCoreVSimdGprGprGprIntrinsicHB;
+}
+
+let TargetPrefix = "riscv" in {
+defm add : CoreVSimdBinary<true>;
+def int_riscv_cv_simd_add_h : ScalarCoreVSimdGprGprImmIntrinsic;
+defm sub : CoreVSimdBinary<true>;
+def int_riscv_cv_simd_sub_h : ScalarCoreVSimdGprGprImmIntrinsic;
+defm avg : CoreVSimdBinary;
+defm avgu : CoreVSimdBinary;
+defm min : CoreVSimdBinary;
+defm minu : CoreVSimdBinary;
+defm max : CoreVSimdBinary;
+defm maxu : CoreVSimdBinary;
+defm srl : CoreVSimdBinary;
+defm sra : CoreVSimdBinary;
+defm sll : CoreVSimdBinary;
+defm or : CoreVSimdBinary;
+defm xor : CoreVSimdBinary;
+defm and : CoreVSimdBinary;
+
+defm abs : ScalarCoreVSimdGprIntrinsicHB;
+
+defm dotup : CoreVSimdBinary;
+defm dotusp : CoreVSimdBinary;
+defm dotsp : CoreVSimdBinary;
+defm sdotup : CoreVSimdTernary;
+defm sdotusp : CoreVSimdTernary;
+defm sdotsp : CoreVSimdTernary;
+
+defm extract : ScalarCoreVSimdGprImmIntrinsicHB;
+defm extractu : ScalarCoreVSimdGprImmIntrinsicHB;
+def int_riscv_cv_simd_insert_b : ScalarCoreVSimdGprGprImmIntrinsic;
+def int_riscv_cv_simd_insert_h : ScalarCoreVSimdGprGprImmIntrinsic;
+
+
+defm shuffle : ScalarCoreVSimdGprGprIntrinsicHB;
+def int_riscv_cv_simd_shuffle_sci_h : ScalarCoreVSimdGprImmIntrinsic;
+def int_riscv_cv_simd_shuffle_sci_b : ScalarCoreVSimdGprImmIntrinsic;
+defm shuffle2 : ScalarCoreVSimdGprGprGprIntrinsicHB;
+
+def int_riscv_cv_simd_pack : ScalarCoreVSimdGprGprIntrinsic;
+def int_riscv_cv_simd_pack_h : ScalarCoreVSimdGprGprIntrinsic;
+def int_riscv_cv_simd_packhi_b : ScalarCoreVSimdGprGprGprIntrinsic;
+def int_riscv_cv_simd_packlo_b : ScalarCoreVSimdGprGprGprIntrinsic;
+
+defm cmpeq : CoreVSimdBinary;
+defm cmpne : CoreVSimdBinary;
+defm cmpgt : CoreVSimdBinary;
+defm cmpge : CoreVSimdBinary;
+defm cmplt : CoreVSimdBinary;
+defm cmple : CoreVSimdBinary;
+defm cmpgtu : CoreVSimdBinary;
+defm cmpgeu : CoreVSimdBinary;
+defm cmpltu : CoreVSimdBinary;
+defm cmpleu : CoreVSimdBinary;
+
+def int_riscv_cv_simd_cplxmul_r : ScalarCoreVSimdGprGprGprImmIntrinsic;
+def int_riscv_cv_simd_cplxmul_i : ScalarCoreVSimdGprGprGprImmIntrinsic;
+
+def int_riscv_cv_simd_cplxconj : ScalarCoreVSimdGprIntrinsic;
+
+def int_riscv_cv_simd_subrotmj : ScalarCoreVSimdGprGprImmIntrinsic;
+} // TargetPrefix = "riscv"

llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp

@@ -52,6 +52,7 @@ class RISCVExpandPseudo : public MachineFunctionPass {
                          MachineBasicBlock::iterator MBBI, unsigned Opcode);
   bool expandVSPILL(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
   bool expandVRELOAD(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
+  bool expandCoreVShuffle(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
 };
 
 char RISCVExpandPseudo::ID = 0;
@@ -132,6 +133,8 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
   case RISCV::PseudoVRELOAD7_M1:
   case RISCV::PseudoVRELOAD8_M1:
     return expandVRELOAD(MBB, MBBI);
+  case RISCV::CV_SHUFFLE_SCI_B_PSEUDO:
+    return expandCoreVShuffle(MBB, MBBI);
   }
 
   return false;
@@ -329,6 +332,23 @@ bool RISCVExpandPseudo::expandVRELOAD(MachineBasicBlock &MBB,
   return true;
 }
 
+bool RISCVExpandPseudo::expandCoreVShuffle(MachineBasicBlock &MBB,
+                                           MachineBasicBlock::iterator MBBI) {
+  DebugLoc DL = MBBI->getDebugLoc();
+  Register DstReg = MBBI->getOperand(0).getReg();
+  Register SrcReg = MBBI->getOperand(1).getReg();
+  uint8_t Imm = MBBI->getOperand(2).getImm();
+  const unsigned Opcodes[] = {
+      RISCV::CV_SHUFFLEI0_SCI_B, RISCV::CV_SHUFFLEI1_SCI_B,
+      RISCV::CV_SHUFFLEI2_SCI_B, RISCV::CV_SHUFFLEI3_SCI_B};
+  const MCInstrDesc &Desc = TII->get(Opcodes[Imm >> 6]);
+  BuildMI(MBB, MBBI, DL, Desc, DstReg)
+      .addReg(SrcReg)
+      .addImm(APInt(6, Imm, true).getSExtValue());
+  MBBI->eraseFromParent();
+  return true;
+}
+
 class RISCVPreRAExpandPseudo : public MachineFunctionPass {
 public:
   const RISCVInstrInfo *TII;

llvm/lib/Target/RISCV/RISCVInstrInfoCOREV.td

@@ -69,6 +69,17 @@ def cv_uimm12 : Operand<XLenVT>,
   let OperandNamespace = "RISCVOp";
 }
 
+def cv_tsimm6 : Operand<XLenVT>, TImmLeaf<XLenVT, [{return isInt<6>(Imm);}]> {
+  let ParserMatchClass = SImmAsmOperand<6>;
+  let EncoderMethod = "getImmOpValue";
+  let DecoderMethod = "decodeSImmOperand<6>";
+  let OperandType = "OPERAND_SIMM6";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    return MCOp.evaluateAsConstantImm(Imm) && isUInt<6>(Imm);
+  }];
+  let OperandNamespace = "RISCVOp";
+}
 
 def cv_uimm6 : Operand<XLenVT>, ImmLeaf<XLenVT, [{return isUInt<6>(Imm);}]> {
   let ParserMatchClass = CVUImmAsmOperand<6>;
@@ -82,6 +93,8 @@ def cv_uimm6 : Operand<XLenVT>, ImmLeaf<XLenVT, [{return isUInt<6>(Imm);}]> {
   let OperandNamespace = "RISCVOp";
 }
 
+def cv_imm8: Operand<XLenVT>, TImmLeaf<XLenVT, [{return isUInt<8>(Imm);}]>;
+
 //===----------------------------------------------------------------------===//
 // CORE-V specific instructions
 //===----------------------------------------------------------------------===//
@@ -822,6 +835,200 @@ let Predicates = [HasExtXCoreVMem] in {
 
 }
 
+//===----------------------------------------------------------------------===//
+// Patterns for SIMD operations
+//===----------------------------------------------------------------------===//
+
+class PatCorevGprGpr <string intr, string asm> :
+  PatGprGpr<!cast<Intrinsic>("int_riscv_cv_simd_" # intr), 
+            !cast<RVInst>("CV_" # asm)>;
+
+// Note that rd is the last argument
+class PatCorevGprGprGpr <string intr, string asm> :
+  Pat<(!cast<Intrinsic>("int_riscv_cv_simd_" # intr) GPR:$rs1, GPR:$rs2, GPR:$rd), 
+      (!cast<RVInst>("CV_" # asm) GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+
+class PatCorevGprTImm <string intr, string asm> :
+  PatGprImm<!cast<Intrinsic>("int_riscv_cv_simd_" # intr), 
+            !cast<RVInst>("CV_" # asm), cv_tsimm6>;
+
+class PatCorevGprImm <string intr, string asm> :
+  PatGprImm<!cast<Intrinsic>("int_riscv_cv_simd_" # intr), 
+            !cast<RVInst>("CV_" # asm), simm6>;
+
+class PatCorevGprUImm <string intr, string asm> :
+  PatGprImm<!cast<Intrinsic>("int_riscv_cv_simd_" # intr), 
+            !cast<RVInst>("CV_" # asm), cv_uimm6>;
+
+multiclass PatCorevGprGprHB <string intr> {
+  def : PatCorevGprGpr<intr # "_h", NAME # "_H">;
+  def : PatCorevGprGpr<intr # "_b", NAME # "_B">;
+}
+
+multiclass PatCorevGprGprGprHB <string intr> {
+  def : PatCorevGprGprGpr<intr # "_h", NAME # "_H">;
+  def : PatCorevGprGprGpr<intr # "_b", NAME # "_B">;
+}
+
+multiclass PatCorevGprTImmHB <string intr> {
+  def : PatCorevGprTImm<intr # "_h", NAME # "_H">;
+  def : PatCorevGprTImm<intr # "_b", NAME # "_B">;
+}
+
+multiclass PatCorevGprImmHB <string intr> {
+  def : PatCorevGprImm<intr # "_h", NAME # "_H">;
+  def : PatCorevGprImm<intr # "_b", NAME # "_B">;
+}
+
+multiclass PatCorevGprUImmHB <string intr> {
+  def : PatCorevGprUImm<intr # "_h", NAME # "_H">;
+  def : PatCorevGprUImm<intr # "_b", NAME # "_B">;
+}
+
+class PatCorevGprGprDiv <Intrinsic intr, string asm, int div> : 
+  Pat<(intr GPR:$rs1, GPR:$rs2, (i32 div)),
+      (!cast<RVInst>("CV_" # asm) GPR:$rs1, GPR:$rs2)>;
+
+class PatCorevGprGprGprDiv <Intrinsic intr, string asm, int div> : 
+  Pat<(intr GPR:$rd, GPR:$rs1, GPR:$rs2, (i32 div)),
+      (!cast<RVInst>("CV_" # asm) GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+
+class PatCorevGprShuffle <Intrinsic intr, RVInst inst, int s> : 
+  Pat<(intr GPR:$rs1, (i32 s)),
+      (inst GPR:$rs1, 0)>; // TODO: parse it
+
+multiclass PatCorevGprGprDivAll <Intrinsic intr> {
+  def : PatCorevGprGprDiv<intr, NAME # "_DIV2", 1>;
+  def : PatCorevGprGprDiv<intr, NAME # "_DIV4", 2>;
+  def : PatCorevGprGprDiv<intr, NAME # "_DIV8", 3>;
+}
+
+multiclass PatCorevGprGprGprDivAll <Intrinsic intr> {
+  def : PatCorevGprGprGprDiv<intr, NAME # "_DIV2", 1>;
+  def : PatCorevGprGprGprDiv<intr, NAME # "_DIV4", 2>;
+  def : PatCorevGprGprGprDiv<intr, NAME # "_DIV8", 3>;
+}
+
+class PatCorevGpr <string intr, string asm> :
+  PatGpr<!cast<Intrinsic>("int_riscv_cv_simd_" # intr), 
+            !cast<RVInst>("CV_" # asm)>;
+
+multiclass PatCorevGprHB <string intr> {
+  def : PatCorevGpr<intr # "_h", NAME # "_H">;
+  def : PatCorevGpr<intr # "_b", NAME # "_B">;
+}
+
+multiclass PatCorevBinary <string intr, bit exclude_h = false> {
+  if exclude_h then {
+    def : PatCorevGprGpr<intr # "_b", NAME # "_B">;
+  } else {
+    defm NAME : PatCorevGprGprHB<intr>;
+  }
+  defm NAME # "_SC" : PatCorevGprGprHB<intr # "_sc">;
+  defm NAME # "_SCI" : PatCorevGprImmHB<intr # "_sc">;
+}
+
+multiclass PatCorevBinaryUnsigned <string intr> {
+  defm NAME : PatCorevGprGprHB<intr>;
+  defm NAME # "_SC" : PatCorevGprGprHB<intr # "_sc">;
+  defm NAME # "_SCI" : PatCorevGprUImmHB<intr # "_sc">;
+}
+
+multiclass PatCorevTernary <string intr> {
+  defm NAME : PatCorevGprGprGprHB<intr>;
+  defm NAME # "_SC" : PatCorevGprGprGprHB<intr # "_sc">;
+  def : Pat<(!cast<Intrinsic>("int_riscv_cv_simd_" # intr # "_sc_h") GPR:$rs1, simm6:$rs2, GPR:$rd),
+            (!cast<RVInst>("CV_" # NAME # "_SCI_H") GPR:$rd, GPR:$rs1, simm6:$rs2)>;
+  def : Pat<(!cast<Intrinsic>("int_riscv_cv_simd_" # intr # "_sc_b") GPR:$rs1, simm6:$rs2, GPR:$rd),
+            (!cast<RVInst>("CV_" # NAME # "_SCI_B") GPR:$rd, GPR:$rs1, simm6:$rs2)>;
+}
+
+multiclass PatCorevTernaryUnsigned <string intr> {
+  defm NAME : PatCorevGprGprGprHB<intr>;
+  defm NAME # "_SC" : PatCorevGprGprGprHB<intr # "_sc">;
+  def : Pat<(!cast<Intrinsic>("int_riscv_cv_simd_" # intr # "_sc_h") GPR:$rs1, cv_uimm6:$rs2, GPR:$rd),
+            (!cast<RVInst>("CV_" # NAME # "_SCI_H") GPR:$rd, GPR:$rs1, cv_uimm6:$rs2)>;
+  def : Pat<(!cast<Intrinsic>("int_riscv_cv_simd_" # intr # "_sc_b") GPR:$rs1, cv_uimm6:$rs2, GPR:$rd),
+            (!cast<RVInst>("CV_" # NAME # "_SCI_B") GPR:$rd, GPR:$rs1, cv_uimm6:$rs2)>;
+}
+
+let Predicates = [HasExtXcvsimd, IsRV32] in {
+  defm ADD :  PatCorevBinary<"add", true>;
+  def : PatCorevGprGprDiv<int_riscv_cv_simd_add_h, "ADD_H", 0>;
+  defm SUB :  PatCorevBinary<"sub", true>;
+  def : PatCorevGprGprDiv<int_riscv_cv_simd_sub_h, "SUB_H", 0>;
+  defm AVG :  PatCorevBinary<"avg">;
+  defm AVGU : PatCorevBinary<"avgu">;
+  defm MIN :  PatCorevBinary<"min">;
+  defm MINU : PatCorevBinaryUnsigned<"minu">;
+  defm MAX :  PatCorevBinary<"max">;
+  defm MAXU : PatCorevBinaryUnsigned<"maxu">;
+  defm SRL :  PatCorevBinaryUnsigned<"srl">;
+  defm SRA :  PatCorevBinaryUnsigned<"sra">;
+  defm SLL :  PatCorevBinaryUnsigned<"sll">;
+  defm OR :   PatCorevBinary<"or">;
+  defm XOR :  PatCorevBinary<"xor">;
+  defm AND :  PatCorevBinary<"and">;
+
+  defm ABS : PatCorevGprHB<"abs">;
+
+  defm DOTUP :   PatCorevBinaryUnsigned<"dotup">;
+  defm DOTUSP :  PatCorevBinary<"dotusp">;
+  defm DOTSP :   PatCorevBinary<"dotsp">;
+  defm SDOTUP :  PatCorevTernaryUnsigned<"sdotup">;
+  defm SDOTUSP : PatCorevTernary<"sdotusp">;
+  defm SDOTSP :  PatCorevTernary<"sdotsp">;
+
+  defm EXTRACT :  PatCorevGprTImmHB<"extract">;
+  defm EXTRACTU : PatCorevGprTImmHB<"extractu">;
+  def : Pat<(int_riscv_cv_simd_insert_b GPR:$rd, GPR:$rs1, cv_tsimm6:$imm), 
+            (CV_INSERT_B GPR:$rd, GPR:$rs1, cv_tsimm6:$imm)>;
+  def : Pat<(int_riscv_cv_simd_insert_h GPR:$rd, GPR:$rs1, cv_tsimm6:$imm), 
+            (CV_INSERT_H GPR:$rd, GPR:$rs1, cv_tsimm6:$imm)>;
+
+  defm SHUFFLE : PatCorevGprGprHB<"shuffle">;
+  def : PatCorevGprTImm<"shuffle_sci_h", "SHUFFLE_SCI_H">;
+
+  def CV_SHUFFLE_SCI_B_PSEUDO : Pseudo<(outs GPR:$rd), (ins GPR:$rs, cv_imm8:$imm), []>;
+  def : PatGprImm<int_riscv_cv_simd_shuffle_sci_b, CV_SHUFFLE_SCI_B_PSEUDO, cv_imm8>;
+
+  def : Pat<(int_riscv_cv_simd_shuffle2_h GPR:$rs1, GPR:$rs2, GPR:$rd),
+            (CV_SHUFFLE2_H GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+  def : Pat<(int_riscv_cv_simd_shuffle2_b GPR:$rs1, GPR:$rs2, GPR:$rd),
+            (CV_SHUFFLE2_B GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+
+  def : PatCorevGprGpr<"pack", "PACK">;
+  def : PatCorevGprGpr<"pack_h", "PACK_H">;
+  def : Pat<(int_riscv_cv_simd_packhi_b GPR:$rd, GPR:$rs1, GPR:$rs2), 
+            (CV_PACKHI_B GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+  def : Pat<(int_riscv_cv_simd_packlo_b GPR:$rd, GPR:$rs1, GPR:$rs2), 
+            (CV_PACKLO_B GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+
+  defm CMPEQ :  PatCorevBinary<"cmpeq">;
+  defm CMPNE :  PatCorevBinary<"cmpne">;
+  defm CMPGT :  PatCorevBinary<"cmpgt">;
+  defm CMPGE :  PatCorevBinary<"cmpge">;
+  defm CMPLT :  PatCorevBinary<"cmplt">;
+  defm CMPLE :  PatCorevBinary<"cmple">;
+  defm CMPGTU : PatCorevBinaryUnsigned<"cmpgtu">;
+  defm CMPGEU : PatCorevBinaryUnsigned<"cmpgeu">;
+  defm CMPLTU : PatCorevBinaryUnsigned<"cmpltu">;
+  defm CMPLEU : PatCorevBinaryUnsigned<"cmpleu">;
+
+  def : PatCorevGprGprGprDiv<int_riscv_cv_simd_cplxmul_r, "CPLXMUL_R", 0>;
+  defm CPLXMUL_R : PatCorevGprGprGprDivAll<int_riscv_cv_simd_cplxmul_r>;
+  def : PatCorevGprGprGprDiv<int_riscv_cv_simd_cplxmul_i, "CPLXMUL_I", 0>;
+  defm CPLXMUL_I : PatCorevGprGprGprDivAll<int_riscv_cv_simd_cplxmul_i>;
+
+  def : PatCorevGpr<"cplxconj", "CPLXCONJ">;
+
+  def : PatCorevGprGprDiv<int_riscv_cv_simd_subrotmj, "SUBROTMJ", 0>;
+  defm SUBROTMJ : PatCorevGprGprDivAll<int_riscv_cv_simd_subrotmj>;
+
+  defm ADD : PatCorevGprGprDivAll<int_riscv_cv_simd_add_h>;
+  defm SUB : PatCorevGprGprDivAll<int_riscv_cv_simd_sub_h>;
+}
+
 //===----------------------------------------------------------------------===//
 // Pseudo instructions and patterns for hardware loop generation
 //===----------------------------------------------------------------------===//